Merge tag 'iommu-updates-v6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

+37

Documentation/ABI/testing/sysfs-bus-event_source-devices-iommu

··· 1 + What: /sys/bus/event_source/devices/dmar*/format 2 + Date: Jan 2023 3 + KernelVersion: 6.3 4 + Contact: Kan Liang <kan.liang@linux.intel.com> 5 + Description: Read-only. Attribute group to describe the magic bits 6 + that go into perf_event_attr.config, 7 + perf_event_attr.config1 or perf_event_attr.config2 for 8 + the IOMMU pmu. (See also 9 + ABI/testing/sysfs-bus-event_source-devices-format). 10 + 11 + Each attribute in this group defines a bit range in 12 + perf_event_attr.config, perf_event_attr.config1, 13 + or perf_event_attr.config2. All supported attributes 14 + are listed below (See the VT-d Spec 4.0 for possible 15 + attribute values):: 16 + 17 + event = "config:0-27" - event ID 18 + event_group = "config:28-31" - event group ID 19 + 20 + filter_requester_en = "config1:0" - Enable Requester ID filter 21 + filter_domain_en = "config1:1" - Enable Domain ID filter 22 + filter_pasid_en = "config1:2" - Enable PASID filter 23 + filter_ats_en = "config1:3" - Enable Address Type filter 24 + filter_page_table_en= "config1:4" - Enable Page Table Level filter 25 + filter_requester_id = "config1:16-31" - Requester ID filter 26 + filter_domain = "config1:32-47" - Domain ID filter 27 + filter_pasid = "config2:0-21" - PASID filter 28 + filter_ats = "config2:24-28" - Address Type filter 29 + filter_page_table = "config2:32-36" - Page Table Level filter 30 + 31 + What: /sys/bus/event_source/devices/dmar*/cpumask 32 + Date: Jan 2023 33 + KernelVersion: 6.3 34 + Contact: Kan Liang <kan.liang@linux.intel.com> 35 + Description: Read-only. This file always returns the CPU to which the 36 + IOMMU pmu is bound for access to all IOMMU pmu performance 37 + monitoring events.

+1

Documentation/devicetree/bindings/iommu/apple,dart.yaml

··· 24 24 compatible: 25 25 enum: 26 26 - apple,t8103-dart 27 + - apple,t8110-dart 27 28 - apple,t6000-dart 28 29 29 30 reg:

+57 -9

Documentation/devicetree/bindings/iommu/arm,smmu.yaml

··· 36 36 - enum: 37 37 - qcom,qcm2290-smmu-500 38 38 - qcom,qdu1000-smmu-500 39 + - qcom,sa8775p-smmu-500 39 40 - qcom,sc7180-smmu-500 40 41 - qcom,sc7280-smmu-500 41 42 - qcom,sc8180x-smmu-500 42 43 - qcom,sc8280xp-smmu-500 43 44 - qcom,sdm670-smmu-500 44 45 - qcom,sdm845-smmu-500 46 + - qcom,sdx55-smmu-500 47 + - qcom,sdx65-smmu-500 45 48 - qcom,sm6115-smmu-500 49 + - qcom,sm6125-smmu-500 46 50 - qcom,sm6350-smmu-500 47 51 - qcom,sm6375-smmu-500 48 52 - qcom,sm8150-smmu-500 ··· 54 50 - qcom,sm8350-smmu-500 55 51 - qcom,sm8450-smmu-500 56 52 - const: qcom,smmu-500 57 - - const: arm,mmu-500 58 - 59 - - description: Qcom SoCs implementing "arm,mmu-500" (non-qcom implementation) 60 - deprecated: true 61 - items: 62 - - enum: 63 - - qcom,sdx55-smmu-500 64 - - qcom,sdx65-smmu-500 65 53 - const: arm,mmu-500 66 54 67 55 - description: Qcom SoCs implementing "arm,mmu-500" (legacy binding) ··· 80 84 items: 81 85 - enum: 82 86 - qcom,sc7280-smmu-500 87 + - qcom,sm8150-smmu-500 83 88 - qcom,sm8250-smmu-500 84 89 - const: qcom,adreno-smmu 85 90 - const: arm,mmu-500 ··· 198 201 maxItems: 7 199 202 200 203 power-domains: 201 - maxItems: 1 204 + minItems: 1 205 + maxItems: 3 202 206 203 207 nvidia,memory-controller: 204 208 description: | ··· 363 365 the smmu ptw 364 366 - description: interface clock required to access smmu's registers 365 367 through the TCU's programming interface. 368 + 369 + # Disallow clocks for all other platforms with specific compatibles 370 + - if: 371 + properties: 372 + compatible: 373 + contains: 374 + enum: 375 + - cavium,smmu-v2 376 + - marvell,ap806-smmu-500 377 + - nvidia,smmu-500 378 + - qcom,qcm2290-smmu-500 379 + - qcom,qdu1000-smmu-500 380 + - qcom,sa8775p-smmu-500 381 + - qcom,sc7180-smmu-500 382 + - qcom,sc8180x-smmu-500 383 + - qcom,sc8280xp-smmu-500 384 + - qcom,sdm670-smmu-500 385 + - qcom,sdm845-smmu-500 386 + - qcom,sdx55-smmu-500 387 + - qcom,sdx65-smmu-500 388 + - qcom,sm6115-smmu-500 389 + - qcom,sm6125-smmu-500 390 + - qcom,sm6350-smmu-500 391 + - qcom,sm6375-smmu-500 392 + - qcom,sm8350-smmu-500 393 + - qcom,sm8450-smmu-500 394 + then: 395 + properties: 396 + clock-names: false 397 + clocks: false 398 + 399 + - if: 400 + properties: 401 + compatible: 402 + contains: 403 + const: qcom,sm6375-smmu-500 404 + then: 405 + properties: 406 + power-domains: 407 + items: 408 + - description: SNoC MMU TBU RT GDSC 409 + - description: SNoC MMU TBU NRT GDSC 410 + - description: SNoC TURING MMU TBU0 GDSC 411 + 412 + required: 413 + - power-domains 414 + else: 415 + properties: 416 + power-domains: 417 + maxItems: 1 366 418 367 419 examples: 368 420 - |+

+1

Documentation/devicetree/bindings/iommu/qcom,iommu.txt

··· 10 10 - compatible : Should be one of: 11 11 12 12 "qcom,msm8916-iommu" 13 + "qcom,msm8953-iommu" 13 14 14 15 Followed by "qcom,msm-iommu-v1". 15 16

+1

Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml

··· 49 49 - enum: 50 50 - renesas,ipmmu-r8a779a0 # R-Car V3U 51 51 - renesas,ipmmu-r8a779f0 # R-Car S4-8 52 + - renesas,ipmmu-r8a779g0 # R-Car V4H 52 53 - const: renesas,rcar-gen4-ipmmu-vmsa # R-Car Gen4 53 54 54 55 reg:

+85 -4

Documentation/devicetree/bindings/reserved-memory/reserved-memory.yaml

··· 52 52 Address and Length pairs. Specifies regions of memory that are 53 53 acceptable to allocate from. 54 54 55 + iommu-addresses: 56 + $ref: /schemas/types.yaml#/definitions/phandle-array 57 + description: > 58 + A list of phandle and specifier pairs that describe static IO virtual 59 + address space mappings and carveouts associated with a given reserved 60 + memory region. The phandle in the first cell refers to the device for 61 + which the mapping or carveout is to be created. 62 + 63 + The specifier consists of an address/size pair and denotes the IO 64 + virtual address range of the region for the given device. The exact 65 + format depends on the values of the "#address-cells" and "#size-cells" 66 + properties of the device referenced via the phandle. 67 + 68 + When used in combination with a "reg" property, an IOVA mapping is to 69 + be established for this memory region. One example where this can be 70 + useful is to create an identity mapping for physical memory that the 71 + firmware has configured some hardware to access (such as a bootsplash 72 + framebuffer). 73 + 74 + If no "reg" property is specified, the "iommu-addresses" property 75 + defines carveout regions in the IOVA space for the given device. This 76 + can be useful if a certain memory region should not be mapped through 77 + the IOMMU. 78 + 55 79 no-map: 56 80 type: boolean 57 81 description: > ··· 113 89 - no-map 114 90 115 91 oneOf: 116 - - required: 117 - - reg 92 + - oneOf: 93 + - required: 94 + - reg 118 95 119 - - required: 120 - - size 96 + - required: 97 + - size 98 + 99 + - oneOf: 100 + # IOMMU reservations 101 + - required: 102 + - iommu-addresses 103 + 104 + # IOMMU mappings 105 + - required: 106 + - reg 107 + - iommu-addresses 121 108 122 109 additionalProperties: true 123 110 111 + examples: 112 + - | 113 + / { 114 + compatible = "foo"; 115 + model = "foo"; 116 + 117 + #address-cells = <2>; 118 + #size-cells = <2>; 119 + 120 + reserved-memory { 121 + #address-cells = <2>; 122 + #size-cells = <2>; 123 + ranges; 124 + 125 + adsp_resv: reservation-adsp { 126 + /* 127 + * Restrict IOVA mappings for ADSP buffers to the 512 MiB region 128 + * from 0x40000000 - 0x5fffffff. Anything outside is reserved by 129 + * the ADSP for I/O memory and private memory allocations. 130 + */ 131 + iommu-addresses = <&adsp 0x0 0x00000000 0x00 0x40000000>, 132 + <&adsp 0x0 0x60000000 0xff 0xa0000000>; 133 + }; 134 + 135 + fb: framebuffer@90000000 { 136 + reg = <0x0 0x90000000 0x0 0x00800000>; 137 + iommu-addresses = <&dc0 0x0 0x90000000 0x0 0x00800000>; 138 + }; 139 + }; 140 + 141 + bus@0 { 142 + #address-cells = <1>; 143 + #size-cells = <1>; 144 + ranges = <0x0 0x0 0x0 0x40000000>; 145 + 146 + adsp: adsp@2990000 { 147 + reg = <0x2990000 0x2000>; 148 + memory-region = <&adsp_resv>; 149 + }; 150 + 151 + dc0: display@15200000 { 152 + reg = <0x15200000 0x10000>; 153 + memory-region = <&fb>; 154 + }; 155 + }; 156 + }; 124 157 ...

-1

MAINTAINERS

··· 10354 10354 S: Supported 10355 10355 T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git 10356 10356 F: drivers/iommu/intel/ 10357 - F: include/linux/intel-svm.h 10358 10357 10359 10358 INTEL IPU3 CSI-2 CIO2 DRIVER 10360 10359 M: Yong Zhi <yong.zhi@intel.com>

+7 -4

arch/arm/mm/dma-mapping.c

··· 984 984 985 985 len = (j - i) << PAGE_SHIFT; 986 986 ret = iommu_map(mapping->domain, iova, phys, len, 987 - __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs)); 987 + __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs), 988 + GFP_KERNEL); 988 989 if (ret < 0) 989 990 goto fail; 990 991 iova += len; ··· 1208 1207 1209 1208 prot = __dma_info_to_prot(dir, attrs); 1210 1209 1211 - ret = iommu_map(mapping->domain, iova, phys, len, prot); 1210 + ret = iommu_map(mapping->domain, iova, phys, len, prot, 1211 + GFP_KERNEL); 1212 1212 if (ret < 0) 1213 1213 goto fail; 1214 1214 count += len >> PAGE_SHIFT; ··· 1381 1379 1382 1380 prot = __dma_info_to_prot(dir, attrs); 1383 1381 1384 - ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1382 + ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 1383 + prot, GFP_KERNEL); 1385 1384 if (ret < 0) 1386 1385 goto fail; 1387 1386 ··· 1446 1443 1447 1444 prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; 1448 1445 1449 - ret = iommu_map(mapping->domain, dma_addr, addr, len, prot); 1446 + ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); 1450 1447 if (ret < 0) 1451 1448 goto fail; 1452 1449

+3 -2

arch/s390/include/asm/pci_dma.h

··· 186 186 187 187 /* Prototypes */ 188 188 void dma_free_seg_table(unsigned long); 189 - unsigned long *dma_alloc_cpu_table(void); 189 + unsigned long *dma_alloc_cpu_table(gfp_t gfp); 190 190 void dma_cleanup_tables(unsigned long *); 191 - unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr); 191 + unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr, 192 + gfp_t gfp); 192 193 void dma_update_cpu_trans(unsigned long *entry, phys_addr_t page_addr, int flags); 193 194 194 195 extern const struct dma_map_ops s390_pci_dma_ops;

+17 -14

arch/s390/pci/pci_dma.c

··· 27 27 zdev->iommu_pages * PAGE_SIZE); 28 28 } 29 29 30 - unsigned long *dma_alloc_cpu_table(void) 30 + unsigned long *dma_alloc_cpu_table(gfp_t gfp) 31 31 { 32 32 unsigned long *table, *entry; 33 33 34 - table = kmem_cache_alloc(dma_region_table_cache, GFP_ATOMIC); 34 + table = kmem_cache_alloc(dma_region_table_cache, gfp); 35 35 if (!table) 36 36 return NULL; 37 37 ··· 45 45 kmem_cache_free(dma_region_table_cache, table); 46 46 } 47 47 48 - static unsigned long *dma_alloc_page_table(void) 48 + static unsigned long *dma_alloc_page_table(gfp_t gfp) 49 49 { 50 50 unsigned long *table, *entry; 51 51 52 - table = kmem_cache_alloc(dma_page_table_cache, GFP_ATOMIC); 52 + table = kmem_cache_alloc(dma_page_table_cache, gfp); 53 53 if (!table) 54 54 return NULL; 55 55 ··· 63 63 kmem_cache_free(dma_page_table_cache, table); 64 64 } 65 65 66 - static unsigned long *dma_get_seg_table_origin(unsigned long *rtep) 66 + static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp) 67 67 { 68 68 unsigned long old_rte, rte; 69 69 unsigned long *sto; ··· 72 72 if (reg_entry_isvalid(rte)) { 73 73 sto = get_rt_sto(rte); 74 74 } else { 75 - sto = dma_alloc_cpu_table(); 75 + sto = dma_alloc_cpu_table(gfp); 76 76 if (!sto) 77 77 return NULL; 78 78 ··· 90 90 return sto; 91 91 } 92 92 93 - static unsigned long *dma_get_page_table_origin(unsigned long *step) 93 + static unsigned long *dma_get_page_table_origin(unsigned long *step, gfp_t gfp) 94 94 { 95 95 unsigned long old_ste, ste; 96 96 unsigned long *pto; ··· 99 99 if (reg_entry_isvalid(ste)) { 100 100 pto = get_st_pto(ste); 101 101 } else { 102 - pto = dma_alloc_page_table(); 102 + pto = dma_alloc_page_table(gfp); 103 103 if (!pto) 104 104 return NULL; 105 105 set_st_pto(&ste, virt_to_phys(pto)); ··· 116 116 return pto; 117 117 } 118 118 119 - unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr) 119 + unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr, 120 + gfp_t gfp) 120 121 { 121 122 unsigned long *sto, *pto; 122 123 unsigned int rtx, sx, px; 123 124 124 125 rtx = calc_rtx(dma_addr); 125 - sto = dma_get_seg_table_origin(&rto[rtx]); 126 + sto = dma_get_seg_table_origin(&rto[rtx], gfp); 126 127 if (!sto) 127 128 return NULL; 128 129 129 130 sx = calc_sx(dma_addr); 130 - pto = dma_get_page_table_origin(&sto[sx]); 131 + pto = dma_get_page_table_origin(&sto[sx], gfp); 131 132 if (!pto) 132 133 return NULL; 133 134 ··· 171 170 return -EINVAL; 172 171 173 172 for (i = 0; i < nr_pages; i++) { 174 - entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr); 173 + entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr, 174 + GFP_ATOMIC); 175 175 if (!entry) { 176 176 rc = -ENOMEM; 177 177 goto undo_cpu_trans; ··· 188 186 while (i-- > 0) { 189 187 page_addr -= PAGE_SIZE; 190 188 dma_addr -= PAGE_SIZE; 191 - entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr); 189 + entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr, 190 + GFP_ATOMIC); 192 191 if (!entry) 193 192 break; 194 193 dma_update_cpu_trans(entry, page_addr, flags); ··· 579 576 580 577 spin_lock_init(&zdev->iommu_bitmap_lock); 581 578 582 - zdev->dma_table = dma_alloc_cpu_table(); 579 + zdev->dma_table = dma_alloc_cpu_table(GFP_KERNEL); 583 580 if (!zdev->dma_table) { 584 581 rc = -ENOMEM; 585 582 goto out;

+2 -1

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

··· 475 475 u32 offset = (r->offset + i) << imem->iommu_pgshift; 476 476 477 477 ret = iommu_map(imem->domain, offset, node->dma_addrs[i], 478 - PAGE_SIZE, IOMMU_READ | IOMMU_WRITE); 478 + PAGE_SIZE, IOMMU_READ | IOMMU_WRITE, 479 + GFP_KERNEL); 479 480 if (ret < 0) { 480 481 nvkm_error(subdev, "IOMMU mapping failure: %d\n", ret); 481 482

+1 -1

drivers/gpu/drm/tegra/drm.c

··· 1057 1057 1058 1058 *dma = iova_dma_addr(&tegra->carveout.domain, alloc); 1059 1059 err = iommu_map(tegra->domain, *dma, virt_to_phys(virt), 1060 - size, IOMMU_READ | IOMMU_WRITE); 1060 + size, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); 1061 1061 if (err < 0) 1062 1062 goto free_iova; 1063 1063

+1 -1

drivers/gpu/host1x/cdma.c

··· 105 105 106 106 pb->dma = iova_dma_addr(&host1x->iova, alloc); 107 107 err = iommu_map(host1x->domain, pb->dma, pb->phys, size, 108 - IOMMU_READ); 108 + IOMMU_READ, GFP_KERNEL); 109 109 if (err) 110 110 goto iommu_free_iova; 111 111 } else {

+4 -4

drivers/infiniband/hw/usnic/usnic_uiom.c

··· 276 276 size = pa_end - pa_start + PAGE_SIZE; 277 277 usnic_dbg("va 0x%lx pa %pa size 0x%zx flags 0x%x", 278 278 va_start, &pa_start, size, flags); 279 - err = iommu_map_atomic(pd->domain, va_start, 280 - pa_start, size, flags); 279 + err = iommu_map(pd->domain, va_start, pa_start, 280 + size, flags, GFP_KERNEL); 281 281 if (err) { 282 282 usnic_err("Failed to map va 0x%lx pa %pa size 0x%zx with err %d\n", 283 283 va_start, &pa_start, size, err); ··· 293 293 size = pa - pa_start + PAGE_SIZE; 294 294 usnic_dbg("va 0x%lx pa %pa size 0x%zx flags 0x%x\n", 295 295 va_start, &pa_start, size, flags); 296 - err = iommu_map_atomic(pd->domain, va_start, 297 - pa_start, size, flags); 296 + err = iommu_map(pd->domain, va_start, pa_start, 297 + size, flags, GFP_KERNEL); 298 298 if (err) { 299 299 usnic_err("Failed to map va 0x%lx pa %pa size 0x%zx with err %d\n", 300 300 va_start, &pa_start, size, err);

+12 -6

drivers/iommu/Kconfig

··· 32 32 config IOMMU_IO_PGTABLE_LPAE 33 33 bool "ARMv7/v8 Long Descriptor Format" 34 34 select IOMMU_IO_PGTABLE 35 - depends on ARM || ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64) 35 + depends on ARM || ARM64 || COMPILE_TEST 36 + depends on !GENERIC_ATOMIC64 # for cpmxchg64() 36 37 help 37 38 Enable support for the ARM long descriptor pagetable format. 38 39 This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page ··· 71 70 config IOMMU_IO_PGTABLE_DART 72 71 bool "Apple DART Formats" 73 72 select IOMMU_IO_PGTABLE 74 - depends on ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64) 73 + depends on ARM64 || COMPILE_TEST 74 + depends on !GENERIC_ATOMIC64 # for cpmxchg64() 75 75 help 76 76 Enable support for the Apple DART pagetable formats. These include 77 77 the t8020 and t6000/t8110 DART formats used in Apple M1/M2 family ··· 286 284 287 285 config IPMMU_VMSA 288 286 bool "Renesas VMSA-compatible IPMMU" 289 - depends on ARCH_RENESAS || (COMPILE_TEST && !GENERIC_ATOMIC64) 287 + depends on ARCH_RENESAS || COMPILE_TEST 288 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 290 289 select IOMMU_API 291 290 select IOMMU_IO_PGTABLE_LPAE 292 291 select ARM_DMA_USE_IOMMU ··· 307 304 308 305 config APPLE_DART 309 306 tristate "Apple DART IOMMU Support" 310 - depends on ARCH_APPLE || (COMPILE_TEST && !GENERIC_ATOMIC64) 307 + depends on ARCH_APPLE || COMPILE_TEST 308 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART 311 309 select IOMMU_API 312 310 select IOMMU_IO_PGTABLE_DART 313 311 default ARCH_APPLE ··· 323 319 # ARM IOMMU support 324 320 config ARM_SMMU 325 321 tristate "ARM Ltd. System MMU (SMMU) Support" 326 - depends on ARM64 || ARM || (COMPILE_TEST && !GENERIC_ATOMIC64) 322 + depends on ARM64 || ARM || COMPILE_TEST 323 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 327 324 select IOMMU_API 328 325 select IOMMU_IO_PGTABLE_LPAE 329 326 select ARM_DMA_USE_IOMMU if ARM ··· 471 466 config QCOM_IOMMU 472 467 # Note: iommu drivers cannot (yet?) be built as modules 473 468 bool "Qualcomm IOMMU Support" 474 - depends on ARCH_QCOM || (COMPILE_TEST && !GENERIC_ATOMIC64) 469 + depends on ARCH_QCOM || COMPILE_TEST 470 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 475 471 select QCOM_SCM 476 472 select IOMMU_API 477 473 select IOMMU_IO_PGTABLE_LPAE

+15 -1

drivers/iommu/amd/init.c

··· 3475 3475 return 1; 3476 3476 } 3477 3477 3478 + #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN) 3479 + 3478 3480 static int __init parse_ivrs_acpihid(char *str) 3479 3481 { 3480 3482 u32 seg = 0, bus, dev, fn; 3481 3483 char *hid, *uid, *p, *addr; 3482 - char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0}; 3484 + char acpiid[ACPIID_LEN] = {0}; 3483 3485 int i; 3484 3486 3485 3487 addr = strchr(str, '@'); 3486 3488 if (!addr) { 3489 + addr = strchr(str, '='); 3490 + if (!addr) 3491 + goto not_found; 3492 + 3493 + ++addr; 3494 + 3495 + if (strlen(addr) > ACPIID_LEN) 3496 + goto not_found; 3497 + 3487 3498 if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 || 3488 3499 sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) { 3489 3500 pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n", ··· 3506 3495 3507 3496 /* We have the '@', make it the terminator to get just the acpiid */ 3508 3497 *addr++ = 0; 3498 + 3499 + if (strlen(str) > ACPIID_LEN + 1) 3500 + goto not_found; 3509 3501 3510 3502 if (sscanf(str, "=%s", acpiid) != 1) 3511 3503 goto not_found;

+50 -39

drivers/iommu/amd/iommu.c

··· 558 558 * prevent logging it. 559 559 */ 560 560 if (IS_IOMMU_MEM_TRANSACTION(flags)) { 561 + /* Device not attached to domain properly */ 562 + if (dev_data->domain == NULL) { 563 + pr_err_ratelimited("Event logged [Device not attached to domain properly]\n"); 564 + pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n", 565 + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), 566 + PCI_FUNC(devid), domain_id); 567 + goto out; 568 + } 569 + 561 570 if (!report_iommu_fault(&dev_data->domain->domain, 562 571 &pdev->dev, address, 563 572 IS_WRITE_REQUEST(flags) ? ··· 676 667 event[0], event[1], event[2], event[3]); 677 668 } 678 669 679 - memset(__evt, 0, 4 * sizeof(u32)); 670 + /* 671 + * To detect the hardware errata 732 we need to clear the 672 + * entry back to zero. This issue does not exist on SNP 673 + * enabled system. Also this buffer is not writeable on 674 + * SNP enabled system. 675 + */ 676 + if (!amd_iommu_snp_en) 677 + memset(__evt, 0, 4 * sizeof(u32)); 680 678 } 681 679 682 680 static void iommu_poll_events(struct amd_iommu *iommu) ··· 752 736 entry[1] = raw[1]; 753 737 754 738 /* 755 - * To detect the hardware bug we need to clear the entry 756 - * back to zero. 739 + * To detect the hardware errata 733 we need to clear the 740 + * entry back to zero. This issue does not exist on SNP 741 + * enabled system. Also this buffer is not writeable on 742 + * SNP enabled system. 757 743 */ 758 - raw[0] = raw[1] = 0UL; 744 + if (!amd_iommu_snp_en) 745 + raw[0] = raw[1] = 0UL; 759 746 760 747 /* Update head pointer of hardware ring-buffer */ 761 748 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; ··· 1721 1702 /* Only allow access to user-accessible pages */ 1722 1703 ret = pci_enable_pasid(pdev, 0); 1723 1704 if (ret) 1724 - goto out_err; 1705 + return ret; 1725 1706 1726 1707 /* First reset the PRI state of the device */ 1727 1708 ret = pci_reset_pri(pdev); 1728 1709 if (ret) 1729 - goto out_err; 1710 + goto out_err_pasid; 1730 1711 1731 1712 /* Enable PRI */ 1732 1713 /* FIXME: Hardcode number of outstanding requests for now */ 1733 1714 ret = pci_enable_pri(pdev, 32); 1734 1715 if (ret) 1735 - goto out_err; 1716 + goto out_err_pasid; 1736 1717 1737 1718 ret = pci_enable_ats(pdev, PAGE_SHIFT); 1738 1719 if (ret) 1739 - goto out_err; 1720 + goto out_err_pri; 1740 1721 1741 1722 return 0; 1742 1723 1743 - out_err: 1724 + out_err_pri: 1744 1725 pci_disable_pri(pdev); 1726 + 1727 + out_err_pasid: 1745 1728 pci_disable_pasid(pdev); 1746 1729 1747 1730 return ret; ··· 2093 2072 if (ret) 2094 2073 goto out_err; 2095 2074 2075 + /* No need to allocate io pgtable ops in passthrough mode */ 2076 + if (type == IOMMU_DOMAIN_IDENTITY) 2077 + return domain; 2078 + 2096 2079 pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain); 2097 2080 if (!pgtbl_ops) { 2098 2081 domain_id_free(domain->id); ··· 2151 2126 protection_domain_free(domain); 2152 2127 } 2153 2128 2154 - static void amd_iommu_detach_device(struct iommu_domain *dom, 2155 - struct device *dev) 2156 - { 2157 - struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2158 - struct amd_iommu *iommu; 2159 - 2160 - if (!check_device(dev)) 2161 - return; 2162 - 2163 - if (dev_data->domain != NULL) 2164 - detach_device(dev); 2165 - 2166 - iommu = rlookup_amd_iommu(dev); 2167 - if (!iommu) 2168 - return; 2169 - 2170 - #ifdef CONFIG_IRQ_REMAP 2171 - if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && 2172 - (dom->type == IOMMU_DOMAIN_UNMANAGED)) 2173 - dev_data->use_vapic = 0; 2174 - #endif 2175 - 2176 - iommu_completion_wait(iommu); 2177 - } 2178 - 2179 2129 static int amd_iommu_attach_device(struct iommu_domain *dom, 2180 2130 struct device *dev) 2181 2131 { ··· 2158 2158 struct protection_domain *domain = to_pdomain(dom); 2159 2159 struct amd_iommu *iommu = rlookup_amd_iommu(dev); 2160 2160 int ret; 2161 + 2162 + /* 2163 + * Skip attach device to domain if new domain is same as 2164 + * devices current domain 2165 + */ 2166 + if (dev_data->domain == domain) 2167 + return 0; 2161 2168 2162 2169 dev_data->defer_attach = false; 2163 2170 ··· 2394 2387 return 0; 2395 2388 2396 2389 /* 2397 - * Do not identity map IOMMUv2 capable devices when memory encryption is 2398 - * active, because some of those devices (AMD GPUs) don't have the 2399 - * encryption bit in their DMA-mask and require remapping. 2390 + * Do not identity map IOMMUv2 capable devices when: 2391 + * - memory encryption is active, because some of those devices 2392 + * (AMD GPUs) don't have the encryption bit in their DMA-mask 2393 + * and require remapping. 2394 + * - SNP is enabled, because it prohibits DTE[Mode]=0. 2400 2395 */ 2401 - if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT) && dev_data->iommu_v2) 2396 + if (dev_data->iommu_v2 && 2397 + !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && 2398 + !amd_iommu_snp_en) { 2402 2399 return IOMMU_DOMAIN_IDENTITY; 2400 + } 2403 2401 2404 2402 return 0; 2405 2403 } ··· 2428 2416 .def_domain_type = amd_iommu_def_domain_type, 2429 2417 .default_domain_ops = &(const struct iommu_domain_ops) { 2430 2418 .attach_dev = amd_iommu_attach_device, 2431 - .detach_dev = amd_iommu_detach_device, 2432 2419 .map_pages = amd_iommu_map_pages, 2433 2420 .unmap_pages = amd_iommu_unmap_pages, 2434 2421 .iotlb_sync_map = amd_iommu_iotlb_sync_map,

+488 -138

drivers/iommu/apple-dart.c

··· 34 34 35 35 #include "dma-iommu.h" 36 36 37 - #define DART_MAX_STREAMS 16 37 + #define DART_MAX_STREAMS 256 38 38 #define DART_MAX_TTBR 4 39 39 #define MAX_DARTS_PER_DEVICE 2 40 40 41 - #define DART_STREAM_ALL 0xffff 41 + /* Common registers */ 42 42 43 43 #define DART_PARAMS1 0x00 44 - #define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24) 44 + #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) 45 45 46 46 #define DART_PARAMS2 0x04 47 - #define DART_PARAMS_BYPASS_SUPPORT BIT(0) 47 + #define DART_PARAMS2_BYPASS_SUPPORT BIT(0) 48 48 49 - #define DART_STREAM_COMMAND 0x20 50 - #define DART_STREAM_COMMAND_BUSY BIT(2) 51 - #define DART_STREAM_COMMAND_INVALIDATE BIT(20) 49 + /* T8020/T6000 registers */ 52 50 53 - #define DART_STREAM_SELECT 0x34 51 + #define DART_T8020_STREAM_COMMAND 0x20 52 + #define DART_T8020_STREAM_COMMAND_BUSY BIT(2) 53 + #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) 54 54 55 - #define DART_ERROR 0x40 56 - #define DART_ERROR_STREAM GENMASK(27, 24) 57 - #define DART_ERROR_CODE GENMASK(11, 0) 58 - #define DART_ERROR_FLAG BIT(31) 55 + #define DART_T8020_STREAM_SELECT 0x34 59 56 60 - #define DART_ERROR_READ_FAULT BIT(4) 61 - #define DART_ERROR_WRITE_FAULT BIT(3) 62 - #define DART_ERROR_NO_PTE BIT(2) 63 - #define DART_ERROR_NO_PMD BIT(1) 64 - #define DART_ERROR_NO_TTBR BIT(0) 57 + #define DART_T8020_ERROR 0x40 58 + #define DART_T8020_ERROR_STREAM GENMASK(27, 24) 59 + #define DART_T8020_ERROR_CODE GENMASK(11, 0) 60 + #define DART_T8020_ERROR_FLAG BIT(31) 65 61 66 - #define DART_CONFIG 0x60 67 - #define DART_CONFIG_LOCK BIT(15) 62 + #define DART_T8020_ERROR_READ_FAULT BIT(4) 63 + #define DART_T8020_ERROR_WRITE_FAULT BIT(3) 64 + #define DART_T8020_ERROR_NO_PTE BIT(2) 65 + #define DART_T8020_ERROR_NO_PMD BIT(1) 66 + #define DART_T8020_ERROR_NO_TTBR BIT(0) 67 + 68 + #define DART_T8020_CONFIG 0x60 69 + #define DART_T8020_CONFIG_LOCK BIT(15) 68 70 69 71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 70 72 71 - #define DART_ERROR_ADDR_HI 0x54 72 - #define DART_ERROR_ADDR_LO 0x50 73 + #define DART_T8020_ERROR_ADDR_HI 0x54 74 + #define DART_T8020_ERROR_ADDR_LO 0x50 73 75 74 - #define DART_STREAMS_ENABLE 0xfc 76 + #define DART_T8020_STREAMS_ENABLE 0xfc 75 77 76 - #define DART_TCR(sid) (0x100 + 4 * (sid)) 77 - #define DART_TCR_TRANSLATE_ENABLE BIT(7) 78 - #define DART_TCR_BYPASS0_ENABLE BIT(8) 79 - #define DART_TCR_BYPASS1_ENABLE BIT(12) 78 + #define DART_T8020_TCR 0x100 79 + #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) 80 + #define DART_T8020_TCR_BYPASS_DART BIT(8) 81 + #define DART_T8020_TCR_BYPASS_DAPF BIT(12) 80 82 81 - #define DART_TTBR(sid, idx) (0x200 + 16 * (sid) + 4 * (idx)) 82 - #define DART_TTBR_VALID BIT(31) 83 - #define DART_TTBR_SHIFT 12 83 + #define DART_T8020_TTBR 0x200 84 + #define DART_T8020_TTBR_VALID BIT(31) 85 + #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0 86 + #define DART_T8020_TTBR_SHIFT 12 87 + 88 + /* T8110 registers */ 89 + 90 + #define DART_T8110_PARAMS3 0x08 91 + #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) 92 + #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) 93 + #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) 94 + #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) 95 + 96 + #define DART_T8110_PARAMS4 0x0c 97 + #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) 98 + #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) 99 + 100 + #define DART_T8110_TLB_CMD 0x80 101 + #define DART_T8110_TLB_CMD_BUSY BIT(31) 102 + #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) 103 + #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 104 + #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 105 + #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) 106 + 107 + #define DART_T8110_ERROR 0x100 108 + #define DART_T8110_ERROR_STREAM GENMASK(27, 20) 109 + #define DART_T8110_ERROR_CODE GENMASK(14, 0) 110 + #define DART_T8110_ERROR_FLAG BIT(31) 111 + 112 + #define DART_T8110_ERROR_MASK 0x104 113 + 114 + #define DART_T8110_ERROR_READ_FAULT BIT(5) 115 + #define DART_T8110_ERROR_WRITE_FAULT BIT(4) 116 + #define DART_T8110_ERROR_NO_PTE BIT(3) 117 + #define DART_T8110_ERROR_NO_PMD BIT(2) 118 + #define DART_T8110_ERROR_NO_PGD BIT(1) 119 + #define DART_T8110_ERROR_NO_TTBR BIT(0) 120 + 121 + #define DART_T8110_ERROR_ADDR_LO 0x170 122 + #define DART_T8110_ERROR_ADDR_HI 0x174 123 + 124 + #define DART_T8110_PROTECT 0x200 125 + #define DART_T8110_UNPROTECT 0x204 126 + #define DART_T8110_PROTECT_LOCK 0x208 127 + #define DART_T8110_PROTECT_TTBR_TCR BIT(0) 128 + 129 + #define DART_T8110_ENABLE_STREAMS 0xc00 130 + #define DART_T8110_DISABLE_STREAMS 0xc20 131 + 132 + #define DART_T8110_TCR 0x1000 133 + #define DART_T8110_TCR_REMAP GENMASK(11, 8) 134 + #define DART_T8110_TCR_REMAP_EN BIT(7) 135 + #define DART_T8110_TCR_BYPASS_DAPF BIT(2) 136 + #define DART_T8110_TCR_BYPASS_DART BIT(1) 137 + #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) 138 + 139 + #define DART_T8110_TTBR 0x1400 140 + #define DART_T8110_TTBR_VALID BIT(0) 141 + #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2 142 + #define DART_T8110_TTBR_SHIFT 14 143 + 144 + #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) 145 + 146 + #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ 147 + (((dart)->hw->ttbr_count * (sid)) << 2) + \ 148 + ((idx) << 2)) 149 + 150 + struct apple_dart_stream_map; 151 + 152 + enum dart_type { 153 + DART_T8020, 154 + DART_T6000, 155 + DART_T8110, 156 + }; 84 157 85 158 struct apple_dart_hw { 159 + enum dart_type type; 160 + irqreturn_t (*irq_handler)(int irq, void *dev); 161 + int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); 162 + 86 163 u32 oas; 87 164 enum io_pgtable_fmt fmt; 165 + 166 + int max_sid_count; 167 + 168 + u64 lock; 169 + u64 lock_bit; 170 + 171 + u64 error; 172 + 173 + u64 enable_streams; 174 + 175 + u64 tcr; 176 + u64 tcr_enabled; 177 + u64 tcr_disabled; 178 + u64 tcr_bypass; 179 + 180 + u64 ttbr; 181 + u64 ttbr_valid; 182 + u64 ttbr_addr_field_shift; 183 + u64 ttbr_shift; 184 + int ttbr_count; 88 185 }; 89 186 90 187 /* ··· 212 115 213 116 spinlock_t lock; 214 117 118 + u32 ias; 119 + u32 oas; 215 120 u32 pgsize; 121 + u32 num_streams; 216 122 u32 supports_bypass : 1; 217 123 u32 force_bypass : 1; 218 124 219 125 struct iommu_group *sid2group[DART_MAX_STREAMS]; 220 126 struct iommu_device iommu; 127 + 128 + u32 save_tcr[DART_MAX_STREAMS]; 129 + u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; 221 130 }; 222 131 223 132 /* ··· 243 140 */ 244 141 struct apple_dart_stream_map { 245 142 struct apple_dart *dart; 246 - unsigned long sidmap; 143 + DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); 247 144 }; 248 145 struct apple_dart_atomic_stream_map { 249 146 struct apple_dart *dart; 250 - atomic64_t sidmap; 147 + atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; 251 148 }; 252 149 253 150 /* ··· 305 202 static void 306 203 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) 307 204 { 205 + struct apple_dart *dart = stream_map->dart; 308 206 int sid; 309 207 310 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 311 - writel(DART_TCR_TRANSLATE_ENABLE, 312 - stream_map->dart->regs + DART_TCR(sid)); 208 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 209 + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); 313 210 } 314 211 315 212 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) 316 213 { 214 + struct apple_dart *dart = stream_map->dart; 317 215 int sid; 318 216 319 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 320 - writel(0, stream_map->dart->regs + DART_TCR(sid)); 217 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 218 + writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); 321 219 } 322 220 323 221 static void 324 222 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) 325 223 { 224 + struct apple_dart *dart = stream_map->dart; 326 225 int sid; 327 226 328 227 WARN_ON(!stream_map->dart->supports_bypass); 329 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 330 - writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE, 331 - stream_map->dart->regs + DART_TCR(sid)); 228 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 229 + writel(dart->hw->tcr_bypass, 230 + dart->regs + DART_TCR(dart, sid)); 332 231 } 333 232 334 233 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, 335 234 u8 idx, phys_addr_t paddr) 336 235 { 236 + struct apple_dart *dart = stream_map->dart; 337 237 int sid; 338 238 339 - WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1)); 340 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 341 - writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT), 342 - stream_map->dart->regs + DART_TTBR(sid, idx)); 239 + WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); 240 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 241 + writel(dart->hw->ttbr_valid | 242 + (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, 243 + dart->regs + DART_TTBR(dart, sid, idx)); 343 244 } 344 245 345 246 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, 346 247 u8 idx) 347 248 { 249 + struct apple_dart *dart = stream_map->dart; 348 250 int sid; 349 251 350 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 351 - writel(0, stream_map->dart->regs + DART_TTBR(sid, idx)); 252 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 253 + writel(0, dart->regs + DART_TTBR(dart, sid, idx)); 352 254 } 353 255 354 256 static void ··· 361 253 { 362 254 int i; 363 255 364 - for (i = 0; i < DART_MAX_TTBR; ++i) 256 + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) 365 257 apple_dart_hw_clear_ttbr(stream_map, i); 366 258 } 367 259 368 260 static int 369 - apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, 261 + apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, 370 262 u32 command) 371 263 { 372 264 unsigned long flags; ··· 375 267 376 268 spin_lock_irqsave(&stream_map->dart->lock, flags); 377 269 378 - writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT); 379 - writel(command, stream_map->dart->regs + DART_STREAM_COMMAND); 270 + writel(stream_map->sidmap[0], stream_map->dart->regs + DART_T8020_STREAM_SELECT); 271 + writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); 380 272 381 273 ret = readl_poll_timeout_atomic( 382 - stream_map->dart->regs + DART_STREAM_COMMAND, command_reg, 383 - !(command_reg & DART_STREAM_COMMAND_BUSY), 1, 274 + stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, 275 + !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, 384 276 DART_STREAM_COMMAND_BUSY_TIMEOUT); 385 277 386 278 spin_unlock_irqrestore(&stream_map->dart->lock, flags); ··· 388 280 if (ret) { 389 281 dev_err(stream_map->dart->dev, 390 282 "busy bit did not clear after command %x for streams %lx\n", 391 - command, stream_map->sidmap); 283 + command, stream_map->sidmap[0]); 392 284 return ret; 393 285 } 394 286 ··· 396 288 } 397 289 398 290 static int 399 - apple_dart_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 291 + apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, 292 + u32 command) 400 293 { 401 - return apple_dart_hw_stream_command(stream_map, 402 - DART_STREAM_COMMAND_INVALIDATE); 294 + struct apple_dart *dart = stream_map->dart; 295 + unsigned long flags; 296 + int ret = 0; 297 + int sid; 298 + 299 + spin_lock_irqsave(&dart->lock, flags); 300 + 301 + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { 302 + u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | 303 + FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); 304 + writel(val, dart->regs + DART_T8110_TLB_CMD); 305 + 306 + ret = readl_poll_timeout_atomic( 307 + dart->regs + DART_T8110_TLB_CMD, val, 308 + !(val & DART_T8110_TLB_CMD_BUSY), 1, 309 + DART_STREAM_COMMAND_BUSY_TIMEOUT); 310 + 311 + if (ret) 312 + break; 313 + 314 + } 315 + 316 + spin_unlock_irqrestore(&dart->lock, flags); 317 + 318 + if (ret) { 319 + dev_err(stream_map->dart->dev, 320 + "busy bit did not clear after command %x for stream %d\n", 321 + command, sid); 322 + return ret; 323 + } 324 + 325 + return 0; 326 + } 327 + 328 + static int 329 + apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 330 + { 331 + return apple_dart_t8020_hw_stream_command( 332 + stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); 333 + } 334 + 335 + static int 336 + apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 337 + { 338 + return apple_dart_t8110_hw_tlb_command( 339 + stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); 403 340 } 404 341 405 342 static int apple_dart_hw_reset(struct apple_dart *dart) 406 343 { 407 344 u32 config; 408 345 struct apple_dart_stream_map stream_map; 346 + int i; 409 347 410 - config = readl(dart->regs + DART_CONFIG); 411 - if (config & DART_CONFIG_LOCK) { 348 + config = readl(dart->regs + dart->hw->lock); 349 + if (config & dart->hw->lock_bit) { 412 350 dev_err(dart->dev, "DART is locked down until reboot: %08x\n", 413 351 config); 414 352 return -EINVAL; 415 353 } 416 354 417 355 stream_map.dart = dart; 418 - stream_map.sidmap = DART_STREAM_ALL; 356 + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); 357 + bitmap_set(stream_map.sidmap, 0, dart->num_streams); 419 358 apple_dart_hw_disable_dma(&stream_map); 420 359 apple_dart_hw_clear_all_ttbrs(&stream_map); 421 360 422 361 /* enable all streams globally since TCR is used to control isolation */ 423 - writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE); 362 + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) 363 + writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); 424 364 425 365 /* clear any pending errors before the interrupt is unmasked */ 426 - writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR); 366 + writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); 427 367 428 - return apple_dart_hw_invalidate_tlb(&stream_map); 368 + if (dart->hw->type == DART_T8110) 369 + writel(0, dart->regs + DART_T8110_ERROR_MASK); 370 + 371 + return dart->hw->invalidate_tlb(&stream_map); 429 372 } 430 373 431 374 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) 432 375 { 433 - int i; 376 + int i, j; 434 377 struct apple_dart_atomic_stream_map *domain_stream_map; 435 378 struct apple_dart_stream_map stream_map; 436 379 437 380 for_each_stream_map(i, domain, domain_stream_map) { 438 381 stream_map.dart = domain_stream_map->dart; 439 - stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap); 440 - apple_dart_hw_invalidate_tlb(&stream_map); 382 + 383 + for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) 384 + stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); 385 + 386 + stream_map.dart->hw->invalidate_tlb(&stream_map); 441 387 } 442 388 } 443 389 ··· 561 399 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) 562 400 apple_dart_hw_set_ttbr(stream_map, i, 563 401 pgtbl_cfg->apple_dart_cfg.ttbr[i]); 564 - for (; i < DART_MAX_TTBR; ++i) 402 + for (; i < stream_map->dart->hw->ttbr_count; ++i) 565 403 apple_dart_hw_clear_ttbr(stream_map, i); 566 404 567 405 apple_dart_hw_enable_translation(stream_map); 568 - apple_dart_hw_invalidate_tlb(stream_map); 406 + stream_map->dart->hw->invalidate_tlb(stream_map); 569 407 } 570 408 571 409 static int apple_dart_finalize_domain(struct iommu_domain *domain, ··· 575 413 struct apple_dart *dart = cfg->stream_maps[0].dart; 576 414 struct io_pgtable_cfg pgtbl_cfg; 577 415 int ret = 0; 578 - int i; 416 + int i, j; 579 417 580 418 mutex_lock(&dart_domain->init_lock); 581 419 ··· 584 422 585 423 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 586 424 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; 587 - atomic64_set(&dart_domain->stream_maps[i].sidmap, 588 - cfg->stream_maps[i].sidmap); 425 + for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) 426 + atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], 427 + cfg->stream_maps[i].sidmap[j]); 589 428 } 590 429 591 430 pgtbl_cfg = (struct io_pgtable_cfg){ 592 431 .pgsize_bitmap = dart->pgsize, 593 - .ias = 32, 594 - .oas = dart->hw->oas, 432 + .ias = dart->ias, 433 + .oas = dart->oas, 595 434 .coherent_walk = 1, 596 435 .iommu_dev = dart->dev, 597 436 }; ··· 606 443 607 444 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 608 445 domain->geometry.aperture_start = 0; 609 - domain->geometry.aperture_end = DMA_BIT_MASK(32); 446 + domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(dart->ias); 610 447 domain->geometry.force_aperture = true; 611 448 612 449 dart_domain->finalized = true; ··· 621 458 struct apple_dart_stream_map *master_maps, 622 459 bool add_streams) 623 460 { 624 - int i; 461 + int i, j; 625 462 626 463 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 627 464 if (domain_maps[i].dart != master_maps[i].dart) ··· 631 468 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 632 469 if (!domain_maps[i].dart) 633 470 break; 634 - if (add_streams) 635 - atomic64_or(master_maps[i].sidmap, 636 - &domain_maps[i].sidmap); 637 - else 638 - atomic64_and(~master_maps[i].sidmap, 639 - &domain_maps[i].sidmap); 471 + for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { 472 + if (add_streams) 473 + atomic_long_or(master_maps[i].sidmap[j], 474 + &domain_maps[i].sidmap[j]); 475 + else 476 + atomic_long_and(~master_maps[i].sidmap[j], 477 + &domain_maps[i].sidmap[j]); 478 + } 640 479 } 641 480 642 481 return 0; ··· 649 484 { 650 485 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps, 651 486 true); 652 - } 653 - 654 - static int apple_dart_domain_remove_streams(struct apple_dart_domain *domain, 655 - struct apple_dart_master_cfg *cfg) 656 - { 657 - return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps, 658 - false); 659 487 } 660 488 661 489 static int apple_dart_attach_dev(struct iommu_domain *domain, ··· 691 533 } 692 534 693 535 return ret; 694 - } 695 - 696 - static void apple_dart_detach_dev(struct iommu_domain *domain, 697 - struct device *dev) 698 - { 699 - int i; 700 - struct apple_dart_stream_map *stream_map; 701 - struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 702 - struct apple_dart_domain *dart_domain = to_dart_domain(domain); 703 - 704 - for_each_stream_map(i, cfg, stream_map) 705 - apple_dart_hw_disable_dma(stream_map); 706 - 707 - if (domain->type == IOMMU_DOMAIN_DMA || 708 - domain->type == IOMMU_DOMAIN_UNMANAGED) 709 - apple_dart_domain_remove_streams(dart_domain, cfg); 710 536 } 711 537 712 538 static struct iommu_device *apple_dart_probe_device(struct device *dev) ··· 779 637 780 638 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 781 639 if (cfg->stream_maps[i].dart == dart) { 782 - cfg->stream_maps[i].sidmap |= 1 << sid; 640 + set_bit(sid, cfg->stream_maps[i].sidmap); 783 641 return 0; 784 642 } 785 643 } 786 644 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 787 645 if (!cfg->stream_maps[i].dart) { 788 646 cfg->stream_maps[i].dart = dart; 789 - cfg->stream_maps[i].sidmap = 1 << sid; 647 + set_bit(sid, cfg->stream_maps[i].sidmap); 790 648 return 0; 791 649 } 792 650 } ··· 805 663 mutex_lock(&apple_dart_groups_lock); 806 664 807 665 for_each_stream_map(i, group_master_cfg, stream_map) 808 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 666 + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 809 667 stream_map->dart->sid2group[sid] = NULL; 810 668 811 669 kfree(iommu_data); 812 670 mutex_unlock(&apple_dart_groups_lock); 671 + } 672 + 673 + static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, 674 + struct apple_dart_master_cfg *src) 675 + { 676 + /* 677 + * We know that this function is only called for groups returned from 678 + * pci_device_group and that all Apple Silicon platforms never spread 679 + * PCIe devices from the same bus across multiple DARTs such that we can 680 + * just assume that both src and dst only have the same single DART. 681 + */ 682 + if (src->stream_maps[1].dart) 683 + return -EINVAL; 684 + if (dst->stream_maps[1].dart) 685 + return -EINVAL; 686 + if (src->stream_maps[0].dart != dst->stream_maps[0].dart) 687 + return -EINVAL; 688 + 689 + bitmap_or(dst->stream_maps[0].sidmap, 690 + dst->stream_maps[0].sidmap, 691 + src->stream_maps[0].sidmap, 692 + dst->stream_maps[0].dart->num_streams); 693 + return 0; 813 694 } 814 695 815 696 static struct iommu_group *apple_dart_device_group(struct device *dev) ··· 847 682 mutex_lock(&apple_dart_groups_lock); 848 683 849 684 for_each_stream_map(i, cfg, stream_map) { 850 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) { 685 + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { 851 686 struct iommu_group *stream_group = 852 687 stream_map->dart->sid2group[sid]; 853 688 ··· 876 711 if (!group) 877 712 goto out; 878 713 879 - group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), GFP_KERNEL); 880 - if (!group_master_cfg) { 881 - iommu_group_put(group); 882 - goto out; 714 + group_master_cfg = iommu_group_get_iommudata(group); 715 + if (group_master_cfg) { 716 + int ret; 717 + 718 + ret = apple_dart_merge_master_cfg(group_master_cfg, cfg); 719 + if (ret) { 720 + dev_err(dev, "Failed to merge DART IOMMU grups.\n"); 721 + iommu_group_put(group); 722 + res = ERR_PTR(ret); 723 + goto out; 724 + } 725 + } else { 726 + group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), 727 + GFP_KERNEL); 728 + if (!group_master_cfg) { 729 + iommu_group_put(group); 730 + goto out; 731 + } 732 + 733 + iommu_group_set_iommudata(group, group_master_cfg, 734 + apple_dart_release_group); 883 735 } 884 736 885 - iommu_group_set_iommudata(group, group_master_cfg, 886 - apple_dart_release_group); 887 - 888 737 for_each_stream_map(i, cfg, stream_map) 889 - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) 738 + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 890 739 stream_map->dart->sid2group[sid] = group; 891 740 892 741 res = group; ··· 959 780 .owner = THIS_MODULE, 960 781 .default_domain_ops = &(const struct iommu_domain_ops) { 961 782 .attach_dev = apple_dart_attach_dev, 962 - .detach_dev = apple_dart_detach_dev, 963 783 .map_pages = apple_dart_map_pages, 964 784 .unmap_pages = apple_dart_unmap_pages, 965 785 .flush_iotlb_all = apple_dart_flush_iotlb_all, ··· 969 791 } 970 792 }; 971 793 972 - static irqreturn_t apple_dart_irq(int irq, void *dev) 794 + static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) 973 795 { 974 796 struct apple_dart *dart = dev; 975 797 const char *fault_name = NULL; 976 - u32 error = readl(dart->regs + DART_ERROR); 977 - u32 error_code = FIELD_GET(DART_ERROR_CODE, error); 978 - u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO); 979 - u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI); 798 + u32 error = readl(dart->regs + DART_T8020_ERROR); 799 + u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); 800 + u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); 801 + u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); 980 802 u64 addr = addr_lo | (((u64)addr_hi) << 32); 981 - u8 stream_idx = FIELD_GET(DART_ERROR_STREAM, error); 803 + u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); 982 804 983 - if (!(error & DART_ERROR_FLAG)) 805 + if (!(error & DART_T8020_ERROR_FLAG)) 984 806 return IRQ_NONE; 985 807 986 808 /* there should only be a single bit set but let's use == to be sure */ 987 - if (error_code == DART_ERROR_READ_FAULT) 809 + if (error_code == DART_T8020_ERROR_READ_FAULT) 988 810 fault_name = "READ FAULT"; 989 - else if (error_code == DART_ERROR_WRITE_FAULT) 811 + else if (error_code == DART_T8020_ERROR_WRITE_FAULT) 990 812 fault_name = "WRITE FAULT"; 991 - else if (error_code == DART_ERROR_NO_PTE) 813 + else if (error_code == DART_T8020_ERROR_NO_PTE) 992 814 fault_name = "NO PTE FOR IOVA"; 993 - else if (error_code == DART_ERROR_NO_PMD) 815 + else if (error_code == DART_T8020_ERROR_NO_PMD) 994 816 fault_name = "NO PMD FOR IOVA"; 995 - else if (error_code == DART_ERROR_NO_TTBR) 817 + else if (error_code == DART_T8020_ERROR_NO_TTBR) 996 818 fault_name = "NO TTBR FOR IOVA"; 997 819 else 998 820 fault_name = "unknown"; ··· 1002 824 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 1003 825 error, stream_idx, error_code, fault_name, addr); 1004 826 1005 - writel(error, dart->regs + DART_ERROR); 827 + writel(error, dart->regs + DART_T8020_ERROR); 828 + return IRQ_HANDLED; 829 + } 830 + 831 + static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) 832 + { 833 + struct apple_dart *dart = dev; 834 + const char *fault_name = NULL; 835 + u32 error = readl(dart->regs + DART_T8110_ERROR); 836 + u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); 837 + u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); 838 + u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); 839 + u64 addr = addr_lo | (((u64)addr_hi) << 32); 840 + u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); 841 + 842 + if (!(error & DART_T8110_ERROR_FLAG)) 843 + return IRQ_NONE; 844 + 845 + /* there should only be a single bit set but let's use == to be sure */ 846 + if (error_code == DART_T8110_ERROR_READ_FAULT) 847 + fault_name = "READ FAULT"; 848 + else if (error_code == DART_T8110_ERROR_WRITE_FAULT) 849 + fault_name = "WRITE FAULT"; 850 + else if (error_code == DART_T8110_ERROR_NO_PTE) 851 + fault_name = "NO PTE FOR IOVA"; 852 + else if (error_code == DART_T8110_ERROR_NO_PMD) 853 + fault_name = "NO PMD FOR IOVA"; 854 + else if (error_code == DART_T8110_ERROR_NO_PGD) 855 + fault_name = "NO PGD FOR IOVA"; 856 + else if (error_code == DART_T8110_ERROR_NO_TTBR) 857 + fault_name = "NO TTBR FOR IOVA"; 858 + else 859 + fault_name = "unknown"; 860 + 861 + dev_err_ratelimited( 862 + dart->dev, 863 + "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 864 + error, stream_idx, error_code, fault_name, addr); 865 + 866 + writel(error, dart->regs + DART_T8110_ERROR); 1006 867 return IRQ_HANDLED; 1007 868 } 1008 869 1009 870 static int apple_dart_probe(struct platform_device *pdev) 1010 871 { 1011 872 int ret; 1012 - u32 dart_params[2]; 873 + u32 dart_params[4]; 1013 874 struct resource *res; 1014 875 struct apple_dart *dart; 1015 876 struct device *dev = &pdev->dev; ··· 1083 866 if (ret) 1084 867 return ret; 1085 868 869 + dart_params[0] = readl(dart->regs + DART_PARAMS1); 870 + dart_params[1] = readl(dart->regs + DART_PARAMS2); 871 + dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); 872 + dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; 873 + 874 + switch (dart->hw->type) { 875 + case DART_T8020: 876 + case DART_T6000: 877 + dart->ias = 32; 878 + dart->oas = dart->hw->oas; 879 + dart->num_streams = dart->hw->max_sid_count; 880 + break; 881 + 882 + case DART_T8110: 883 + dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); 884 + dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); 885 + dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); 886 + dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); 887 + dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); 888 + break; 889 + } 890 + 891 + if (dart->num_streams > DART_MAX_STREAMS) { 892 + dev_err(&pdev->dev, "Too many streams (%d > %d)\n", 893 + dart->num_streams, DART_MAX_STREAMS); 894 + ret = -EINVAL; 895 + goto err_clk_disable; 896 + } 897 + 898 + dart->force_bypass = dart->pgsize > PAGE_SIZE; 899 + 1086 900 ret = apple_dart_hw_reset(dart); 1087 901 if (ret) 1088 902 goto err_clk_disable; 1089 903 1090 - dart_params[0] = readl(dart->regs + DART_PARAMS1); 1091 - dart_params[1] = readl(dart->regs + DART_PARAMS2); 1092 - dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]); 1093 - dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT; 1094 - dart->force_bypass = dart->pgsize > PAGE_SIZE; 1095 - 1096 - ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, 904 + ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, 1097 905 "apple-dart fault handler", dart); 1098 906 if (ret) 1099 907 goto err_clk_disable; ··· 1136 894 1137 895 dev_info( 1138 896 &pdev->dev, 1139 - "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n", 1140 - dart->pgsize, dart->supports_bypass, dart->force_bypass); 897 + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", 898 + dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass); 1141 899 return 0; 1142 900 1143 901 err_sysfs_remove: ··· 1166 924 } 1167 925 1168 926 static const struct apple_dart_hw apple_dart_hw_t8103 = { 927 + .type = DART_T8020, 928 + .irq_handler = apple_dart_t8020_irq, 929 + .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1169 930 .oas = 36, 1170 931 .fmt = APPLE_DART, 932 + .max_sid_count = 16, 933 + 934 + .enable_streams = DART_T8020_STREAMS_ENABLE, 935 + .lock = DART_T8020_CONFIG, 936 + .lock_bit = DART_T8020_CONFIG_LOCK, 937 + 938 + .error = DART_T8020_ERROR, 939 + 940 + .tcr = DART_T8020_TCR, 941 + .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 942 + .tcr_disabled = 0, 943 + .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 944 + 945 + .ttbr = DART_T8020_TTBR, 946 + .ttbr_valid = DART_T8020_TTBR_VALID, 947 + .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 948 + .ttbr_shift = DART_T8020_TTBR_SHIFT, 949 + .ttbr_count = 4, 1171 950 }; 1172 951 static const struct apple_dart_hw apple_dart_hw_t6000 = { 952 + .type = DART_T6000, 953 + .irq_handler = apple_dart_t8020_irq, 954 + .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1173 955 .oas = 42, 1174 956 .fmt = APPLE_DART2, 957 + .max_sid_count = 16, 958 + 959 + .enable_streams = DART_T8020_STREAMS_ENABLE, 960 + .lock = DART_T8020_CONFIG, 961 + .lock_bit = DART_T8020_CONFIG_LOCK, 962 + 963 + .error = DART_T8020_ERROR, 964 + 965 + .tcr = DART_T8020_TCR, 966 + .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 967 + .tcr_disabled = 0, 968 + .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 969 + 970 + .ttbr = DART_T8020_TTBR, 971 + .ttbr_valid = DART_T8020_TTBR_VALID, 972 + .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 973 + .ttbr_shift = DART_T8020_TTBR_SHIFT, 974 + .ttbr_count = 4, 1175 975 }; 976 + 977 + static const struct apple_dart_hw apple_dart_hw_t8110 = { 978 + .type = DART_T8110, 979 + .irq_handler = apple_dart_t8110_irq, 980 + .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, 981 + .fmt = APPLE_DART2, 982 + .max_sid_count = 256, 983 + 984 + .enable_streams = DART_T8110_ENABLE_STREAMS, 985 + .lock = DART_T8110_PROTECT, 986 + .lock_bit = DART_T8110_PROTECT_TTBR_TCR, 987 + 988 + .error = DART_T8110_ERROR, 989 + 990 + .tcr = DART_T8110_TCR, 991 + .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, 992 + .tcr_disabled = 0, 993 + .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, 994 + 995 + .ttbr = DART_T8110_TTBR, 996 + .ttbr_valid = DART_T8110_TTBR_VALID, 997 + .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, 998 + .ttbr_shift = DART_T8110_TTBR_SHIFT, 999 + .ttbr_count = 1, 1000 + }; 1001 + 1002 + static __maybe_unused int apple_dart_suspend(struct device *dev) 1003 + { 1004 + struct apple_dart *dart = dev_get_drvdata(dev); 1005 + unsigned int sid, idx; 1006 + 1007 + for (sid = 0; sid < dart->num_streams; sid++) { 1008 + dart->save_tcr[sid] = readl_relaxed(dart->regs + DART_TCR(dart, sid)); 1009 + for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1010 + dart->save_ttbr[sid][idx] = 1011 + readl(dart->regs + DART_TTBR(dart, sid, idx)); 1012 + } 1013 + 1014 + return 0; 1015 + } 1016 + 1017 + static __maybe_unused int apple_dart_resume(struct device *dev) 1018 + { 1019 + struct apple_dart *dart = dev_get_drvdata(dev); 1020 + unsigned int sid, idx; 1021 + int ret; 1022 + 1023 + ret = apple_dart_hw_reset(dart); 1024 + if (ret) { 1025 + dev_err(dev, "Failed to reset DART on resume\n"); 1026 + return ret; 1027 + } 1028 + 1029 + for (sid = 0; sid < dart->num_streams; sid++) { 1030 + for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1031 + writel(dart->save_ttbr[sid][idx], 1032 + dart->regs + DART_TTBR(dart, sid, idx)); 1033 + writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); 1034 + } 1035 + 1036 + return 0; 1037 + } 1038 + 1039 + DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); 1176 1040 1177 1041 static const struct of_device_id apple_dart_of_match[] = { 1178 1042 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, 1043 + { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, 1179 1044 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, 1180 1045 {}, 1181 1046 }; ··· 1293 944 .name = "apple-dart", 1294 945 .of_match_table = apple_dart_of_match, 1295 946 .suppress_bind_attrs = true, 947 + .pm = pm_sleep_ptr(&apple_dart_pm_ops), 1296 948 }, 1297 949 .probe = apple_dart_probe, 1298 950 .remove = apple_dart_remove,

+2

drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c

··· 250 250 { .compatible = "qcom,sc7280-mdss" }, 251 251 { .compatible = "qcom,sc7280-mss-pil" }, 252 252 { .compatible = "qcom,sc8180x-mdss" }, 253 + { .compatible = "qcom,sc8280xp-mdss" }, 254 + { .compatible = "qcom,sm8150-mdss" }, 253 255 { .compatible = "qcom,sm8250-mdss" }, 254 256 { .compatible = "qcom,sdm845-mdss" }, 255 257 { .compatible = "qcom,sdm845-mss-pil" },

-23

drivers/iommu/arm/arm-smmu/qcom_iommu.c

··· 387 387 return 0; 388 388 } 389 389 390 - static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) 391 - { 392 - struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); 393 - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 394 - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev); 395 - unsigned i; 396 - 397 - if (WARN_ON(!qcom_domain->iommu)) 398 - return; 399 - 400 - pm_runtime_get_sync(qcom_iommu->dev); 401 - for (i = 0; i < fwspec->num_ids; i++) { 402 - struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); 403 - 404 - /* Disable the context bank: */ 405 - iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0); 406 - 407 - ctx->domain = NULL; 408 - } 409 - pm_runtime_put_sync(qcom_iommu->dev); 410 - } 411 - 412 390 static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova, 413 391 phys_addr_t paddr, size_t pgsize, size_t pgcount, 414 392 int prot, gfp_t gfp, size_t *mapped) ··· 561 583 .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 562 584 .default_domain_ops = &(const struct iommu_domain_ops) { 563 585 .attach_dev = qcom_iommu_attach_dev, 564 - .detach_dev = qcom_iommu_detach_dev, 565 586 .map_pages = qcom_iommu_map, 566 587 .unmap_pages = qcom_iommu_unmap, 567 588 .flush_iotlb_all = qcom_iommu_flush_iotlb_all,

+16 -5

drivers/iommu/dma-iommu.c

··· 23 23 #include <linux/memremap.h> 24 24 #include <linux/mm.h> 25 25 #include <linux/mutex.h> 26 + #include <linux/of_iommu.h> 26 27 #include <linux/pci.h> 27 28 #include <linux/scatterlist.h> 28 29 #include <linux/spinlock.h> ··· 392 391 if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) 393 392 iort_iommu_get_resv_regions(dev, list); 394 393 394 + if (dev->of_node) 395 + of_iommu_get_resv_regions(dev, list); 395 396 } 396 397 EXPORT_SYMBOL(iommu_dma_get_resv_regions); 397 398 ··· 716 713 if (!iova) 717 714 return DMA_MAPPING_ERROR; 718 715 719 - if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) { 716 + if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) { 720 717 iommu_dma_free_iova(cookie, iova, size, NULL); 721 718 return DMA_MAPPING_ERROR; 722 719 } ··· 825 822 if (!iova) 826 823 goto out_free_pages; 827 824 828 - if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL)) 825 + /* 826 + * Remove the zone/policy flags from the GFP - these are applied to the 827 + * __iommu_dma_alloc_pages() but are not used for the supporting 828 + * internal allocations that follow. 829 + */ 830 + gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); 831 + 832 + if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, gfp)) 829 833 goto out_free_iova; 830 834 831 835 if (!(ioprot & IOMMU_CACHE)) { ··· 843 833 arch_dma_prep_coherent(sg_page(sg), sg->length); 844 834 } 845 835 846 - ret = iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot); 836 + ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot, 837 + gfp); 847 838 if (ret < 0 || ret < size) 848 839 goto out_free_sg; 849 840 ··· 1292 1281 * We'll leave any physical concatenation to the IOMMU driver's 1293 1282 * implementation - it knows better than we do. 1294 1283 */ 1295 - ret = iommu_map_sg_atomic(domain, iova, sg, nents, prot); 1284 + ret = iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 1296 1285 if (ret < 0 || ret < iova_len) 1297 1286 goto out_free_iova; 1298 1287 ··· 1626 1615 if (!iova) 1627 1616 goto out_free_page; 1628 1617 1629 - if (iommu_map(domain, iova, msi_addr, size, prot)) 1618 + if (iommu_map(domain, iova, msi_addr, size, prot, GFP_KERNEL)) 1630 1619 goto out_free_iova; 1631 1620 1632 1621 INIT_LIST_HEAD(&msi_page->list);

+148 -66

drivers/iommu/exynos-iommu.c

··· 185 185 lv2table_base(sent)) + lv2ent_offset(iova); 186 186 } 187 187 188 - /* 189 - * IOMMU fault information register 190 - */ 191 - struct sysmmu_fault_info { 192 - unsigned int bit; /* bit number in STATUS register */ 193 - unsigned short addr_reg; /* register to read VA fault address */ 188 + struct sysmmu_fault { 189 + sysmmu_iova_t addr; /* IOVA address that caused fault */ 190 + const char *name; /* human readable fault name */ 191 + unsigned int type; /* fault type for report_iommu_fault() */ 192 + }; 193 + 194 + struct sysmmu_v1_fault_info { 195 + unsigned short addr_reg; /* register to read IOVA fault address */ 194 196 const char *name; /* human readable fault name */ 195 197 unsigned int type; /* fault type for report_iommu_fault */ 196 198 }; 197 199 198 - static const struct sysmmu_fault_info sysmmu_faults[] = { 199 - { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, 200 - { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, 201 - { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 202 - { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, 203 - { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 204 - { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 205 - { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 206 - { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 200 + static const struct sysmmu_v1_fault_info sysmmu_v1_faults[] = { 201 + { REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, 202 + { REG_AR_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_READ }, 203 + { REG_AW_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_WRITE }, 204 + { REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, 205 + { REG_AR_FAULT_ADDR, "SECURITY PROTECTION", IOMMU_FAULT_READ }, 206 + { REG_AR_FAULT_ADDR, "ACCESS PROTECTION", IOMMU_FAULT_READ }, 207 + { REG_AW_FAULT_ADDR, "SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 208 + { REG_AW_FAULT_ADDR, "ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 207 209 }; 208 210 209 - static const struct sysmmu_fault_info sysmmu_v5_faults[] = { 210 - { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, 211 - { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, 212 - { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, 213 - { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, 214 - { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, 215 - { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, 216 - { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, 217 - { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, 218 - { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, 219 - { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, 211 + /* SysMMU v5 has the same faults for AR (0..4 bits) and AW (16..20 bits) */ 212 + static const char * const sysmmu_v5_fault_names[] = { 213 + "PTW", 214 + "PAGE", 215 + "MULTI-HIT", 216 + "ACCESS PROTECTION", 217 + "SECURITY PROTECTION" 218 + }; 219 + 220 + static const char * const sysmmu_v7_fault_names[] = { 221 + "PTW", 222 + "PAGE", 223 + "ACCESS PROTECTION", 224 + "RESERVED" 220 225 }; 221 226 222 227 /* ··· 251 246 struct iommu_domain domain; /* generic domain data structure */ 252 247 }; 253 248 249 + struct sysmmu_drvdata; 250 + 254 251 /* 255 252 * SysMMU version specific data. Contains offsets for the registers which can 256 253 * be found in different SysMMU variants, but have different offset values. 254 + * Also contains version specific callbacks to abstract the hardware. 257 255 */ 258 256 struct sysmmu_variant { 259 257 u32 pt_base; /* page table base address (physical) */ ··· 267 259 u32 flush_end; /* end address of range invalidation */ 268 260 u32 int_status; /* interrupt status information */ 269 261 u32 int_clear; /* clear the interrupt */ 262 + u32 fault_va; /* IOVA address that caused fault */ 263 + u32 fault_info; /* fault transaction info */ 264 + 265 + int (*get_fault_info)(struct sysmmu_drvdata *data, unsigned int itype, 266 + struct sysmmu_fault *fault); 270 267 }; 271 268 272 269 /* ··· 306 293 307 294 #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg) 308 295 296 + static int exynos_sysmmu_v1_get_fault_info(struct sysmmu_drvdata *data, 297 + unsigned int itype, 298 + struct sysmmu_fault *fault) 299 + { 300 + const struct sysmmu_v1_fault_info *finfo; 301 + 302 + if (itype >= ARRAY_SIZE(sysmmu_v1_faults)) 303 + return -ENXIO; 304 + 305 + finfo = &sysmmu_v1_faults[itype]; 306 + fault->addr = readl(data->sfrbase + finfo->addr_reg); 307 + fault->name = finfo->name; 308 + fault->type = finfo->type; 309 + 310 + return 0; 311 + } 312 + 313 + static int exynos_sysmmu_v5_get_fault_info(struct sysmmu_drvdata *data, 314 + unsigned int itype, 315 + struct sysmmu_fault *fault) 316 + { 317 + unsigned int addr_reg; 318 + 319 + if (itype < ARRAY_SIZE(sysmmu_v5_fault_names)) { 320 + fault->type = IOMMU_FAULT_READ; 321 + addr_reg = REG_V5_FAULT_AR_VA; 322 + } else if (itype >= 16 && itype <= 20) { 323 + fault->type = IOMMU_FAULT_WRITE; 324 + addr_reg = REG_V5_FAULT_AW_VA; 325 + itype -= 16; 326 + } else { 327 + return -ENXIO; 328 + } 329 + 330 + fault->name = sysmmu_v5_fault_names[itype]; 331 + fault->addr = readl(data->sfrbase + addr_reg); 332 + 333 + return 0; 334 + } 335 + 336 + static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data, 337 + unsigned int itype, 338 + struct sysmmu_fault *fault) 339 + { 340 + u32 info = readl(SYSMMU_REG(data, fault_info)); 341 + 342 + fault->addr = readl(SYSMMU_REG(data, fault_va)); 343 + fault->name = sysmmu_v7_fault_names[itype % 4]; 344 + fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 345 + 346 + return 0; 347 + } 348 + 309 349 /* SysMMU v1..v3 */ 310 350 static const struct sysmmu_variant sysmmu_v1_variant = { 311 351 .flush_all = 0x0c, ··· 366 300 .pt_base = 0x14, 367 301 .int_status = 0x18, 368 302 .int_clear = 0x1c, 303 + 304 + .get_fault_info = exynos_sysmmu_v1_get_fault_info, 369 305 }; 370 306 371 - /* SysMMU v5 and v7 (non-VM capable) */ 307 + /* SysMMU v5 */ 372 308 static const struct sysmmu_variant sysmmu_v5_variant = { 373 309 .pt_base = 0x0c, 374 310 .flush_all = 0x10, ··· 380 312 .flush_end = 0x24, 381 313 .int_status = 0x60, 382 314 .int_clear = 0x64, 315 + 316 + .get_fault_info = exynos_sysmmu_v5_get_fault_info, 383 317 }; 384 318 385 - /* SysMMU v7: VM capable register set */ 319 + /* SysMMU v7: non-VM capable register layout */ 320 + static const struct sysmmu_variant sysmmu_v7_variant = { 321 + .pt_base = 0x0c, 322 + .flush_all = 0x10, 323 + .flush_entry = 0x14, 324 + .flush_range = 0x18, 325 + .flush_start = 0x20, 326 + .flush_end = 0x24, 327 + .int_status = 0x60, 328 + .int_clear = 0x64, 329 + .fault_va = 0x70, 330 + .fault_info = 0x78, 331 + 332 + .get_fault_info = exynos_sysmmu_v7_get_fault_info, 333 + }; 334 + 335 + /* SysMMU v7: VM capable register layout */ 386 336 static const struct sysmmu_variant sysmmu_v7_vm_variant = { 387 337 .pt_base = 0x800c, 388 338 .flush_all = 0x8010, ··· 410 324 .flush_end = 0x8024, 411 325 .int_status = 0x60, 412 326 .int_clear = 0x64, 327 + .fault_va = 0x1000, 328 + .fault_info = 0x1004, 329 + 330 + .get_fault_info = exynos_sysmmu_v7_get_fault_info, 413 331 }; 414 332 415 333 static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) ··· 536 446 if (data->has_vcr) 537 447 data->variant = &sysmmu_v7_vm_variant; 538 448 else 539 - data->variant = &sysmmu_v5_variant; 449 + data->variant = &sysmmu_v7_variant; 540 450 } 541 451 542 452 __sysmmu_disable_clocks(data); 543 453 } 544 454 545 455 static void show_fault_information(struct sysmmu_drvdata *data, 546 - const struct sysmmu_fault_info *finfo, 547 - sysmmu_iova_t fault_addr) 456 + const struct sysmmu_fault *fault) 548 457 { 549 458 sysmmu_pte_t *ent; 550 459 551 - dev_err(data->sysmmu, "%s: %s FAULT occurred at %#x\n", 552 - dev_name(data->master), finfo->name, fault_addr); 460 + dev_err(data->sysmmu, "%s: [%s] %s FAULT occurred at %#x\n", 461 + dev_name(data->master), 462 + fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE", 463 + fault->name, fault->addr); 553 464 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); 554 - ent = section_entry(phys_to_virt(data->pgtable), fault_addr); 465 + ent = section_entry(phys_to_virt(data->pgtable), fault->addr); 555 466 dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); 556 467 if (lv1ent_page(ent)) { 557 - ent = page_entry(ent, fault_addr); 468 + ent = page_entry(ent, fault->addr); 558 469 dev_dbg(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); 559 470 } 560 471 } 561 472 562 473 static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) 563 474 { 564 - /* SYSMMU is in blocked state when interrupt occurred. */ 565 475 struct sysmmu_drvdata *data = dev_id; 566 - const struct sysmmu_fault_info *finfo; 567 - unsigned int i, n, itype; 568 - sysmmu_iova_t fault_addr; 476 + unsigned int itype; 477 + struct sysmmu_fault fault; 569 478 int ret = -ENOSYS; 570 479 571 480 WARN_ON(!data->active); 572 481 573 - if (MMU_MAJ_VER(data->version) < 5) { 574 - finfo = sysmmu_faults; 575 - n = ARRAY_SIZE(sysmmu_faults); 576 - } else { 577 - finfo = sysmmu_v5_faults; 578 - n = ARRAY_SIZE(sysmmu_v5_faults); 579 - } 580 - 581 482 spin_lock(&data->lock); 582 - 583 483 clk_enable(data->clk_master); 584 484 585 485 itype = __ffs(readl(SYSMMU_REG(data, int_status))); 586 - for (i = 0; i < n; i++, finfo++) 587 - if (finfo->bit == itype) 588 - break; 589 - /* unknown/unsupported fault */ 590 - BUG_ON(i == n); 486 + ret = data->variant->get_fault_info(data, itype, &fault); 487 + if (ret) { 488 + dev_err(data->sysmmu, "Unhandled interrupt bit %u\n", itype); 489 + goto out; 490 + } 491 + show_fault_information(data, &fault); 591 492 592 - /* print debug message */ 593 - fault_addr = readl(data->sfrbase + finfo->addr_reg); 594 - show_fault_information(data, finfo, fault_addr); 493 + if (data->domain) { 494 + ret = report_iommu_fault(&data->domain->domain, data->master, 495 + fault.addr, fault.type); 496 + } 497 + if (ret) 498 + panic("Unrecoverable System MMU Fault!"); 595 499 596 - if (data->domain) 597 - ret = report_iommu_fault(&data->domain->domain, 598 - data->master, fault_addr, finfo->type); 599 - /* fault is not recovered by fault handler */ 600 - BUG_ON(ret != 0); 601 - 500 + out: 602 501 writel(1 << itype, SYSMMU_REG(data, int_clear)); 603 502 503 + /* SysMMU is in blocked state when interrupt occurred */ 604 504 sysmmu_unblock(data); 605 - 606 505 clk_disable(data->clk_master); 607 - 608 506 spin_unlock(&data->lock); 609 507 610 508 return IRQ_HANDLED; ··· 1424 1346 struct iommu_group *group = iommu_group_get(dev); 1425 1347 1426 1348 if (group) { 1349 + #ifndef CONFIG_ARM 1427 1350 WARN_ON(owner->domain != 1428 1351 iommu_group_default_domain(group)); 1352 + #endif 1429 1353 exynos_iommu_detach_device(owner->domain, dev); 1430 1354 iommu_group_put(group); 1431 1355 } ··· 1478 1398 static const struct iommu_ops exynos_iommu_ops = { 1479 1399 .domain_alloc = exynos_iommu_domain_alloc, 1480 1400 .device_group = generic_device_group, 1401 + #ifdef CONFIG_ARM 1402 + .set_platform_dma_ops = exynos_iommu_release_device, 1403 + #endif 1481 1404 .probe_device = exynos_iommu_probe_device, 1482 1405 .release_device = exynos_iommu_release_device, 1483 1406 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, 1484 1407 .of_xlate = exynos_iommu_of_xlate, 1485 1408 .default_domain_ops = &(const struct iommu_domain_ops) { 1486 1409 .attach_dev = exynos_iommu_attach_device, 1487 - .detach_dev = exynos_iommu_detach_device, 1488 1410 .map = exynos_iommu_map, 1489 1411 .unmap = exynos_iommu_unmap, 1490 1412 .iova_to_phys = exynos_iommu_iova_to_phys, ··· 1528 1446 1529 1447 return 0; 1530 1448 err_reg_driver: 1531 - platform_driver_unregister(&exynos_sysmmu_driver); 1449 + kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); 1532 1450 err_zero_lv2: 1533 1451 kmem_cache_destroy(lv2table_kmem_cache); 1534 1452 return ret;

+3 -3

drivers/iommu/fsl_pamu_domain.c

··· 283 283 return ret; 284 284 } 285 285 286 - static void fsl_pamu_detach_device(struct iommu_domain *domain, 287 - struct device *dev) 286 + static void fsl_pamu_set_platform_dma(struct device *dev) 288 287 { 288 + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 289 289 struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 290 290 const u32 *prop; 291 291 int len; ··· 452 452 .domain_alloc = fsl_pamu_domain_alloc, 453 453 .probe_device = fsl_pamu_probe_device, 454 454 .device_group = fsl_pamu_device_group, 455 + .set_platform_dma_ops = fsl_pamu_set_platform_dma, 455 456 .default_domain_ops = &(const struct iommu_domain_ops) { 456 457 .attach_dev = fsl_pamu_attach_device, 457 - .detach_dev = fsl_pamu_detach_device, 458 458 .iova_to_phys = fsl_pamu_iova_to_phys, 459 459 .free = fsl_pamu_domain_free, 460 460 }

+11

drivers/iommu/intel/Kconfig

··· 96 96 passing intel_iommu=sm_on to the kernel. If not sure, please use 97 97 the default value. 98 98 99 + config INTEL_IOMMU_PERF_EVENTS 100 + def_bool y 101 + bool "Intel IOMMU performance events" 102 + depends on INTEL_IOMMU && PERF_EVENTS 103 + help 104 + Selecting this option will enable the performance monitoring 105 + infrastructure in the Intel IOMMU. It collects information about 106 + key events occurring during operation of the remapping hardware, 107 + to aid performance tuning and debug. These are available on modern 108 + processors which support Intel VT-d 4.0 and later. 109 + 99 110 endif # INTEL_IOMMU

+1

drivers/iommu/intel/Makefile

··· 6 6 obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o 7 7 obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o 8 8 obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o 9 + obj-$(CONFIG_INTEL_IOMMU_PERF_EVENTS) += perfmon.o

+29 -4

drivers/iommu/intel/dmar.c

··· 34 34 #include "../irq_remapping.h" 35 35 #include "perf.h" 36 36 #include "trace.h" 37 + #include "perfmon.h" 37 38 38 39 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *); 39 40 struct dmar_res_callback { ··· 428 427 memcpy(dmaru->hdr, header, header->length); 429 428 dmaru->reg_base_addr = drhd->address; 430 429 dmaru->segment = drhd->segment; 430 + /* The size of the register set is 2 ^ N 4 KB pages. */ 431 + dmaru->reg_size = 1UL << (drhd->size + 12); 431 432 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */ 432 433 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1), 433 434 ((void *)drhd) + drhd->header.length, ··· 959 956 /** 960 957 * map_iommu: map the iommu's registers 961 958 * @iommu: the iommu to map 962 - * @phys_addr: the physical address of the base resgister 959 + * @drhd: DMA remapping hardware definition structure 963 960 * 964 961 * Memory map the iommu's registers. Start w/ a single page, and 965 962 * possibly expand if that turns out to be insufficent. 966 963 */ 967 - static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) 964 + static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd) 968 965 { 966 + u64 phys_addr = drhd->reg_base_addr; 969 967 int map_size, err=0; 970 968 971 969 iommu->reg_phys = phys_addr; 972 - iommu->reg_size = VTD_PAGE_SIZE; 970 + iommu->reg_size = drhd->reg_size; 973 971 974 972 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { 975 973 pr_err("Can't reserve memory\n"); ··· 1017 1013 goto release; 1018 1014 } 1019 1015 } 1016 + 1017 + if (cap_ecmds(iommu->cap)) { 1018 + int i; 1019 + 1020 + for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) { 1021 + iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG + 1022 + i * DMA_ECMD_REG_STEP); 1023 + } 1024 + } 1025 + 1020 1026 err = 0; 1021 1027 goto out; 1022 1028 ··· 1064 1050 } 1065 1051 sprintf(iommu->name, "dmar%d", iommu->seq_id); 1066 1052 1067 - err = map_iommu(iommu, drhd->reg_base_addr); 1053 + err = map_iommu(iommu, drhd); 1068 1054 if (err) { 1069 1055 pr_err("Failed to map %s\n", iommu->name); 1070 1056 goto error_free_seq_id; ··· 1117 1103 if (sts & DMA_GSTS_QIES) 1118 1104 iommu->gcmd |= DMA_GCMD_QIE; 1119 1105 1106 + if (alloc_iommu_pmu(iommu)) 1107 + pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id); 1108 + 1120 1109 raw_spin_lock_init(&iommu->register_lock); 1121 1110 1122 1111 /* ··· 1144 1127 err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL); 1145 1128 if (err) 1146 1129 goto err_sysfs; 1130 + 1131 + iommu_pmu_register(iommu); 1147 1132 } 1148 1133 1149 1134 drhd->iommu = iommu; ··· 1156 1137 err_sysfs: 1157 1138 iommu_device_sysfs_remove(&iommu->iommu); 1158 1139 err_unmap: 1140 + free_iommu_pmu(iommu); 1159 1141 unmap_iommu(iommu); 1160 1142 error_free_seq_id: 1161 1143 ida_free(&dmar_seq_ids, iommu->seq_id); ··· 1168 1148 static void free_iommu(struct intel_iommu *iommu) 1169 1149 { 1170 1150 if (intel_iommu_enabled && !iommu->drhd->ignored) { 1151 + iommu_pmu_unregister(iommu); 1171 1152 iommu_device_unregister(&iommu->iommu); 1172 1153 iommu_device_sysfs_remove(&iommu->iommu); 1173 1154 } 1155 + 1156 + free_iommu_pmu(iommu); 1174 1157 1175 1158 if (iommu->irq) { 1176 1159 if (iommu->pr_irq) { ··· 1882 1859 return DMAR_FECTL_REG; 1883 1860 else if (iommu->pr_irq == irq) 1884 1861 return DMAR_PECTL_REG; 1862 + else if (iommu->perf_irq == irq) 1863 + return DMAR_PERFINTRCTL_REG; 1885 1864 else 1886 1865 BUG(); 1887 1866 }

+100 -22

drivers/iommu/intel/iommu.c

··· 16 16 #include <linux/crash_dump.h> 17 17 #include <linux/dma-direct.h> 18 18 #include <linux/dmi.h> 19 - #include <linux/intel-svm.h> 20 19 #include <linux/memory.h> 21 20 #include <linux/pci.h> 22 21 #include <linux/pci-ats.h> ··· 29 30 #include "../iommu-sva.h" 30 31 #include "pasid.h" 31 32 #include "cap_audit.h" 33 + #include "perfmon.h" 32 34 33 35 #define ROOT_SIZE VTD_PAGE_SIZE 34 36 #define CONTEXT_SIZE VTD_PAGE_SIZE ··· 362 362 } 363 363 __setup("intel_iommu=", intel_iommu_setup); 364 364 365 - void *alloc_pgtable_page(int node) 365 + void *alloc_pgtable_page(int node, gfp_t gfp) 366 366 { 367 367 struct page *page; 368 368 void *vaddr = NULL; 369 369 370 - page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0); 370 + page = alloc_pages_node(node, gfp | __GFP_ZERO, 0); 371 371 if (page) 372 372 vaddr = page_address(page); 373 373 return vaddr; ··· 612 612 if (!alloc) 613 613 return NULL; 614 614 615 - context = alloc_pgtable_page(iommu->node); 615 + context = alloc_pgtable_page(iommu->node, GFP_ATOMIC); 616 616 if (!context) 617 617 return NULL; 618 618 ··· 908 908 #endif 909 909 910 910 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, 911 - unsigned long pfn, int *target_level) 911 + unsigned long pfn, int *target_level, 912 + gfp_t gfp) 912 913 { 913 914 struct dma_pte *parent, *pte; 914 915 int level = agaw_to_level(domain->agaw); ··· 936 935 if (!dma_pte_present(pte)) { 937 936 uint64_t pteval; 938 937 939 - tmp_page = alloc_pgtable_page(domain->nid); 938 + tmp_page = alloc_pgtable_page(domain->nid, gfp); 940 939 941 940 if (!tmp_page) 942 941 return NULL; ··· 1187 1186 { 1188 1187 struct root_entry *root; 1189 1188 1190 - root = (struct root_entry *)alloc_pgtable_page(iommu->node); 1189 + root = (struct root_entry *)alloc_pgtable_page(iommu->node, GFP_ATOMIC); 1191 1190 if (!root) { 1192 1191 pr_err("Allocating root entry for %s failed\n", 1193 1192 iommu->name); ··· 2151 2150 2152 2151 while (start_pfn <= end_pfn) { 2153 2152 if (!pte) 2154 - pte = pfn_to_dma_pte(domain, start_pfn, &level); 2153 + pte = pfn_to_dma_pte(domain, start_pfn, &level, 2154 + GFP_ATOMIC); 2155 2155 2156 2156 if (dma_pte_present(pte)) { 2157 2157 dma_pte_free_pagetable(domain, start_pfn, ··· 2174 2172 2175 2173 static int 2176 2174 __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 2177 - unsigned long phys_pfn, unsigned long nr_pages, int prot) 2175 + unsigned long phys_pfn, unsigned long nr_pages, int prot, 2176 + gfp_t gfp) 2178 2177 { 2179 2178 struct dma_pte *first_pte = NULL, *pte = NULL; 2180 2179 unsigned int largepage_lvl = 0; ··· 2205 2202 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, 2206 2203 phys_pfn, nr_pages); 2207 2204 2208 - pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl); 2205 + pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl, 2206 + gfp); 2209 2207 if (!pte) 2210 2208 return -ENOMEM; 2211 2209 first_pte = pte; ··· 2372 2368 2373 2369 return __domain_mapping(domain, first_vpfn, 2374 2370 first_vpfn, last_vpfn - first_vpfn + 1, 2375 - DMA_PTE_READ|DMA_PTE_WRITE); 2371 + DMA_PTE_READ|DMA_PTE_WRITE, GFP_KERNEL); 2376 2372 } 2377 2373 2378 2374 static int md_domain_init(struct dmar_domain *domain, int guest_width); ··· 2680 2676 if (!old_ce) 2681 2677 goto out; 2682 2678 2683 - new_ce = alloc_pgtable_page(iommu->node); 2679 + new_ce = alloc_pgtable_page(iommu->node, GFP_KERNEL); 2684 2680 if (!new_ce) 2685 2681 goto out_unmap; 2686 2682 ··· 4009 4005 * is likely to be much lower than the overhead of synchronizing 4010 4006 * the virtual and physical IOMMU page-tables. 4011 4007 */ 4012 - if (cap_caching_mode(iommu->cap)) { 4008 + if (cap_caching_mode(iommu->cap) && 4009 + !first_level_by_default(IOMMU_DOMAIN_DMA)) { 4013 4010 pr_info_once("IOMMU batching disallowed due to virtualization\n"); 4014 4011 iommu_set_dma_strict(); 4015 4012 } ··· 4018 4013 intel_iommu_groups, 4019 4014 "%s", iommu->name); 4020 4015 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL); 4016 + 4017 + iommu_pmu_register(iommu); 4021 4018 } 4022 4019 up_read(&dmar_global_lock); 4023 4020 ··· 4143 4136 domain->max_addr = 0; 4144 4137 4145 4138 /* always allocate the top pgd */ 4146 - domain->pgd = alloc_pgtable_page(domain->nid); 4139 + domain->pgd = alloc_pgtable_page(domain->nid, GFP_ATOMIC); 4147 4140 if (!domain->pgd) 4148 4141 return -ENOMEM; 4149 4142 domain_flush_cache(domain, domain->pgd, PAGE_SIZE); ··· 4305 4298 the low bits of hpa would take us onto the next page */ 4306 4299 size = aligned_nrpages(hpa, size); 4307 4300 return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT, 4308 - hpa >> VTD_PAGE_SHIFT, size, prot); 4301 + hpa >> VTD_PAGE_SHIFT, size, prot, gfp); 4309 4302 } 4310 4303 4311 4304 static int intel_iommu_map_pages(struct iommu_domain *domain, ··· 4340 4333 4341 4334 /* Cope with horrid API which requires us to unmap more than the 4342 4335 size argument if it happens to be a large-page mapping. */ 4343 - BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level)); 4336 + BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level, 4337 + GFP_ATOMIC)); 4344 4338 4345 4339 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level)) 4346 4340 size = VTD_PAGE_SIZE << level_to_offset_bits(level); ··· 4354 4346 if (dmar_domain->max_addr == iova + size) 4355 4347 dmar_domain->max_addr = iova; 4356 4348 4357 - iommu_iotlb_gather_add_page(domain, gather, iova, size); 4349 + /* 4350 + * We do not use page-selective IOTLB invalidation in flush queue, 4351 + * so there is no need to track page and sync iotlb. 4352 + */ 4353 + if (!iommu_iotlb_gather_queued(gather)) 4354 + iommu_iotlb_gather_add_page(domain, gather, iova, size); 4358 4355 4359 4356 return size; 4360 4357 } ··· 4405 4392 int level = 0; 4406 4393 u64 phys = 0; 4407 4394 4408 - pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level); 4395 + pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level, 4396 + GFP_ATOMIC); 4409 4397 if (pte && dma_pte_present(pte)) 4410 4398 phys = dma_pte_addr(pte) + 4411 4399 (iova & (BIT_MASK(level_to_offset_bits(level) + ··· 4656 4642 return -EINVAL; 4657 4643 4658 4644 ret = iopf_queue_add_device(iommu->iopf_queue, dev); 4659 - if (!ret) 4660 - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); 4645 + if (ret) 4646 + return ret; 4647 + 4648 + ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); 4649 + if (ret) 4650 + iopf_queue_remove_device(iommu->iopf_queue, dev); 4661 4651 4662 4652 return ret; 4663 4653 } ··· 4673 4655 int ret; 4674 4656 4675 4657 ret = iommu_unregister_device_fault_handler(dev); 4676 - if (!ret) 4677 - ret = iopf_queue_remove_device(iommu->iopf_queue, dev); 4658 + if (ret) 4659 + return ret; 4660 + 4661 + ret = iopf_queue_remove_device(iommu->iopf_queue, dev); 4662 + if (ret) 4663 + iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); 4678 4664 4679 4665 return ret; 4680 4666 } ··· 5044 5022 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid, 5045 5023 pasid, qdep, address, mask); 5046 5024 } 5025 + } 5026 + 5027 + #define ecmd_get_status_code(res) (((res) & 0xff) >> 1) 5028 + 5029 + /* 5030 + * Function to submit a command to the enhanced command interface. The 5031 + * valid enhanced command descriptions are defined in Table 47 of the 5032 + * VT-d spec. The VT-d hardware implementation may support some but not 5033 + * all commands, which can be determined by checking the Enhanced 5034 + * Command Capability Register. 5035 + * 5036 + * Return values: 5037 + * - 0: Command successful without any error; 5038 + * - Negative: software error value; 5039 + * - Nonzero positive: failure status code defined in Table 48. 5040 + */ 5041 + int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob) 5042 + { 5043 + unsigned long flags; 5044 + u64 res; 5045 + int ret; 5046 + 5047 + if (!cap_ecmds(iommu->cap)) 5048 + return -ENODEV; 5049 + 5050 + raw_spin_lock_irqsave(&iommu->register_lock, flags); 5051 + 5052 + res = dmar_readq(iommu->reg + DMAR_ECRSP_REG); 5053 + if (res & DMA_ECMD_ECRSP_IP) { 5054 + ret = -EBUSY; 5055 + goto err; 5056 + } 5057 + 5058 + /* 5059 + * Unconditionally write the operand B, because 5060 + * - There is no side effect if an ecmd doesn't require an 5061 + * operand B, but we set the register to some value. 5062 + * - It's not invoked in any critical path. The extra MMIO 5063 + * write doesn't bring any performance concerns. 5064 + */ 5065 + dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob); 5066 + dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT)); 5067 + 5068 + IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq, 5069 + !(res & DMA_ECMD_ECRSP_IP), res); 5070 + 5071 + if (res & DMA_ECMD_ECRSP_IP) { 5072 + ret = -ETIMEDOUT; 5073 + goto err; 5074 + } 5075 + 5076 + ret = ecmd_get_status_code(res); 5077 + err: 5078 + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 5079 + 5080 + return ret; 5047 5081 }

+106 -9

drivers/iommu/intel/iommu.h

··· 22 22 #include <linux/ioasid.h> 23 23 #include <linux/bitfield.h> 24 24 #include <linux/xarray.h> 25 + #include <linux/perf_event.h> 25 26 26 27 #include <asm/cacheflush.h> 27 28 #include <asm/iommu.h> ··· 126 125 #define DMAR_MTRR_PHYSMASK8_REG 0x208 127 126 #define DMAR_MTRR_PHYSBASE9_REG 0x210 128 127 #define DMAR_MTRR_PHYSMASK9_REG 0x218 128 + #define DMAR_PERFCAP_REG 0x300 129 + #define DMAR_PERFCFGOFF_REG 0x310 130 + #define DMAR_PERFOVFOFF_REG 0x318 131 + #define DMAR_PERFCNTROFF_REG 0x31c 132 + #define DMAR_PERFINTRSTS_REG 0x324 133 + #define DMAR_PERFINTRCTL_REG 0x328 134 + #define DMAR_PERFEVNTCAP_REG 0x380 135 + #define DMAR_ECMD_REG 0x400 136 + #define DMAR_ECEO_REG 0x408 137 + #define DMAR_ECRSP_REG 0x410 138 + #define DMAR_ECCAP_REG 0x430 129 139 #define DMAR_VCCAP_REG 0xe30 /* Virtual command capability register */ 130 140 #define DMAR_VCMD_REG 0xe00 /* Virtual command register */ 131 141 #define DMAR_VCRSP_REG 0xe10 /* Virtual command response register */ ··· 160 148 */ 161 149 #define cap_esrtps(c) (((c) >> 63) & 1) 162 150 #define cap_esirtps(c) (((c) >> 62) & 1) 151 + #define cap_ecmds(c) (((c) >> 61) & 1) 163 152 #define cap_fl5lp_support(c) (((c) >> 60) & 1) 164 153 #define cap_pi_support(c) (((c) >> 59) & 1) 165 154 #define cap_fl1gp_support(c) (((c) >> 56) & 1) ··· 192 179 * Extended Capability Register 193 180 */ 194 181 195 - #define ecap_rps(e) (((e) >> 49) & 0x1) 182 + #define ecap_pms(e) (((e) >> 51) & 0x1) 183 + #define ecap_rps(e) (((e) >> 49) & 0x1) 196 184 #define ecap_smpwc(e) (((e) >> 48) & 0x1) 197 185 #define ecap_flts(e) (((e) >> 47) & 0x1) 198 186 #define ecap_slts(e) (((e) >> 46) & 0x1) ··· 223 209 #define ecap_dev_iotlb_support(e) (((e) >> 2) & 0x1) 224 210 #define ecap_max_handle_mask(e) (((e) >> 20) & 0xf) 225 211 #define ecap_sc_support(e) (((e) >> 7) & 0x1) /* Snooping Control */ 212 + 213 + /* 214 + * Decoding Perf Capability Register 215 + */ 216 + #define pcap_num_cntr(p) ((p) & 0xffff) 217 + #define pcap_cntr_width(p) (((p) >> 16) & 0x7f) 218 + #define pcap_num_event_group(p) (((p) >> 24) & 0x1f) 219 + #define pcap_filters_mask(p) (((p) >> 32) & 0x1f) 220 + #define pcap_interrupt(p) (((p) >> 50) & 0x1) 221 + /* The counter stride is calculated as 2 ^ (x+10) bytes */ 222 + #define pcap_cntr_stride(p) (1ULL << ((((p) >> 52) & 0x7) + 10)) 223 + 224 + /* 225 + * Decoding Perf Event Capability Register 226 + */ 227 + #define pecap_es(p) ((p) & 0xfffffff) 226 228 227 229 /* Virtual command interface capability */ 228 230 #define vccap_pasid(v) (((v) & DMA_VCS_PAS)) /* PASID allocation */ ··· 311 281 #define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16) 312 282 #define DMA_CCMD_DID(d) ((u64)((d) & 0xffff)) 313 283 284 + /* ECMD_REG */ 285 + #define DMA_MAX_NUM_ECMD 256 286 + #define DMA_MAX_NUM_ECMDCAP (DMA_MAX_NUM_ECMD / 64) 287 + #define DMA_ECMD_REG_STEP 8 288 + #define DMA_ECMD_ENABLE 0xf0 289 + #define DMA_ECMD_DISABLE 0xf1 290 + #define DMA_ECMD_FREEZE 0xf4 291 + #define DMA_ECMD_UNFREEZE 0xf5 292 + #define DMA_ECMD_OA_SHIFT 16 293 + #define DMA_ECMD_ECRSP_IP 0x1 294 + #define DMA_ECMD_ECCAP3 3 295 + #define DMA_ECMD_ECCAP3_ECNTS BIT_ULL(48) 296 + #define DMA_ECMD_ECCAP3_DCNTS BIT_ULL(49) 297 + #define DMA_ECMD_ECCAP3_FCNTS BIT_ULL(52) 298 + #define DMA_ECMD_ECCAP3_UFCNTS BIT_ULL(53) 299 + #define DMA_ECMD_ECCAP3_ESSENTIAL (DMA_ECMD_ECCAP3_ECNTS | \ 300 + DMA_ECMD_ECCAP3_DCNTS | \ 301 + DMA_ECMD_ECCAP3_FCNTS | \ 302 + DMA_ECMD_ECCAP3_UFCNTS) 303 + 314 304 /* FECTL_REG */ 315 305 #define DMA_FECTL_IM (((u32)1) << 31) 316 306 ··· 358 308 #define DMA_PRS_PRO ((u32)2) 359 309 360 310 #define DMA_VCS_PAS ((u64)1) 311 + 312 + /* PERFINTRSTS_REG */ 313 + #define DMA_PERFINTRSTS_PIS ((u32)1) 361 314 362 315 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \ 363 316 do { \ ··· 491 438 int free_cnt; 492 439 }; 493 440 441 + /* Page Request Queue depth */ 442 + #define PRQ_ORDER 4 443 + #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 444 + #define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5) 445 + 494 446 struct dmar_pci_notify_info; 495 447 496 448 #ifdef CONFIG_IRQ_REMAP ··· 612 554 iommu core */ 613 555 }; 614 556 557 + /* 558 + * In theory, the VT-d 4.0 spec can support up to 2 ^ 16 counters. 559 + * But in practice, there are only 14 counters for the existing 560 + * platform. Setting the max number of counters to 64 should be good 561 + * enough for a long time. Also, supporting more than 64 counters 562 + * requires more extras, e.g., extra freeze and overflow registers, 563 + * which is not necessary for now. 564 + */ 565 + #define IOMMU_PMU_IDX_MAX 64 566 + 567 + struct iommu_pmu { 568 + struct intel_iommu *iommu; 569 + u32 num_cntr; /* Number of counters */ 570 + u32 num_eg; /* Number of event group */ 571 + u32 cntr_width; /* Counter width */ 572 + u32 cntr_stride; /* Counter Stride */ 573 + u32 filter; /* Bitmask of filter support */ 574 + void __iomem *base; /* the PerfMon base address */ 575 + void __iomem *cfg_reg; /* counter configuration base address */ 576 + void __iomem *cntr_reg; /* counter 0 address*/ 577 + void __iomem *overflow; /* overflow status register */ 578 + 579 + u64 *evcap; /* Indicates all supported events */ 580 + u32 **cntr_evcap; /* Supported events of each counter. */ 581 + 582 + struct pmu pmu; 583 + DECLARE_BITMAP(used_mask, IOMMU_PMU_IDX_MAX); 584 + struct perf_event *event_list[IOMMU_PMU_IDX_MAX]; 585 + unsigned char irq_name[16]; 586 + }; 587 + 588 + #define IOMMU_IRQ_ID_OFFSET_PRQ (DMAR_UNITS_SUPPORTED) 589 + #define IOMMU_IRQ_ID_OFFSET_PERF (2 * DMAR_UNITS_SUPPORTED) 590 + 615 591 struct intel_iommu { 616 592 void __iomem *reg; /* Pointer to hardware regs, virtual addr */ 617 593 u64 reg_phys; /* physical address of hw register set */ ··· 653 561 u64 cap; 654 562 u64 ecap; 655 563 u64 vccap; 564 + u64 ecmdcap[DMA_MAX_NUM_ECMDCAP]; 656 565 u32 gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */ 657 566 raw_spinlock_t register_lock; /* protect register handling */ 658 567 int seq_id; /* sequence id of the iommu */ 659 568 int agaw; /* agaw of this iommu */ 660 569 int msagaw; /* max sagaw of this iommu */ 661 - unsigned int irq, pr_irq; 570 + unsigned int irq, pr_irq, perf_irq; 662 571 u16 segment; /* PCI segment# */ 663 572 unsigned char name[13]; /* Device Name */ 664 573 ··· 693 600 694 601 struct dmar_drhd_unit *drhd; 695 602 void *perf_statistic; 603 + 604 + struct iommu_pmu *pmu; 696 605 }; 697 606 698 607 /* PCI domain-device relationship */ ··· 832 737 833 738 extern int dmar_ir_support(void); 834 739 835 - void *alloc_pgtable_page(int node); 740 + void *alloc_pgtable_page(int node, gfp_t gfp); 836 741 void free_pgtable_page(void *vaddr); 837 742 void iommu_flush_write_buffer(struct intel_iommu *iommu); 838 743 struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); ··· 851 756 struct rcu_head rcu; 852 757 struct device *dev; 853 758 struct intel_iommu *iommu; 854 - struct iommu_sva sva; 855 - u32 pasid; 856 - int users; 857 759 u16 did; 858 - u16 dev_iotlb:1; 859 760 u16 sid, qdep; 860 761 }; 861 762 862 763 struct intel_svm { 863 764 struct mmu_notifier notifier; 864 765 struct mm_struct *mm; 865 - 866 - unsigned int flags; 867 766 u32 pasid; 868 767 struct list_head devs; 869 768 }; ··· 889 800 extern int intel_iommu_sm; 890 801 extern int iommu_calculate_agaw(struct intel_iommu *iommu); 891 802 extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); 803 + int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob); 804 + 805 + static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu) 806 + { 807 + return (iommu->ecmdcap[DMA_ECMD_ECCAP3] & DMA_ECMD_ECCAP3_ESSENTIAL) == 808 + DMA_ECMD_ECCAP3_ESSENTIAL; 809 + } 810 + 892 811 extern int dmar_disabled; 893 812 extern int intel_iommu_enabled; 894 813 #else

+19 -1

drivers/iommu/intel/pasid.c

··· 128 128 pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); 129 129 info->pasid_table = pasid_table; 130 130 131 + if (!ecap_coherent(info->iommu->ecap)) 132 + clflush_cache_range(pasid_table->table, size); 133 + 131 134 return 0; 132 135 } 133 136 ··· 203 200 retry: 204 201 entries = get_pasid_table_from_pde(&dir[dir_index]); 205 202 if (!entries) { 206 - entries = alloc_pgtable_page(info->iommu->node); 203 + entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC); 207 204 if (!entries) 208 205 return NULL; 209 206 ··· 217 214 (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { 218 215 free_pgtable_page(entries); 219 216 goto retry; 217 + } 218 + if (!ecap_coherent(info->iommu->ecap)) { 219 + clflush_cache_range(entries, VTD_PAGE_SIZE); 220 + clflush_cache_range(&dir[dir_index].val, sizeof(*dir)); 220 221 } 221 222 } 222 223 ··· 369 362 static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) 370 363 { 371 364 pasid_set_bits(&pe->val[1], 1 << 23, value << 23); 365 + } 366 + 367 + /* 368 + * Setup No Execute Enable bit (Bit 133) of a scalable mode PASID 369 + * entry. It is required when XD bit of the first level page table 370 + * entry is about to be set. 371 + */ 372 + static inline void pasid_set_nxe(struct pasid_entry *pe) 373 + { 374 + pasid_set_bits(&pe->val[2], 1 << 5, 1 << 5); 372 375 } 373 376 374 377 /* ··· 574 557 pasid_set_domain_id(pte, did); 575 558 pasid_set_address_width(pte, iommu->agaw); 576 559 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 560 + pasid_set_nxe(pte); 577 561 578 562 /* Setup Present and PASID Granular Transfer Type: */ 579 563 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY);

+877

drivers/iommu/intel/perfmon.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Support Intel IOMMU PerfMon 4 + * Copyright(c) 2023 Intel Corporation. 5 + */ 6 + #define pr_fmt(fmt) "DMAR: " fmt 7 + #define dev_fmt(fmt) pr_fmt(fmt) 8 + 9 + #include <linux/dmar.h> 10 + #include "iommu.h" 11 + #include "perfmon.h" 12 + 13 + PMU_FORMAT_ATTR(event, "config:0-27"); /* ES: Events Select */ 14 + PMU_FORMAT_ATTR(event_group, "config:28-31"); /* EGI: Event Group Index */ 15 + 16 + static struct attribute *iommu_pmu_format_attrs[] = { 17 + &format_attr_event_group.attr, 18 + &format_attr_event.attr, 19 + NULL 20 + }; 21 + 22 + static struct attribute_group iommu_pmu_format_attr_group = { 23 + .name = "format", 24 + .attrs = iommu_pmu_format_attrs, 25 + }; 26 + 27 + /* The available events are added in attr_update later */ 28 + static struct attribute *attrs_empty[] = { 29 + NULL 30 + }; 31 + 32 + static struct attribute_group iommu_pmu_events_attr_group = { 33 + .name = "events", 34 + .attrs = attrs_empty, 35 + }; 36 + 37 + static cpumask_t iommu_pmu_cpu_mask; 38 + 39 + static ssize_t 40 + cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) 41 + { 42 + return cpumap_print_to_pagebuf(true, buf, &iommu_pmu_cpu_mask); 43 + } 44 + static DEVICE_ATTR_RO(cpumask); 45 + 46 + static struct attribute *iommu_pmu_cpumask_attrs[] = { 47 + &dev_attr_cpumask.attr, 48 + NULL 49 + }; 50 + 51 + static struct attribute_group iommu_pmu_cpumask_attr_group = { 52 + .attrs = iommu_pmu_cpumask_attrs, 53 + }; 54 + 55 + static const struct attribute_group *iommu_pmu_attr_groups[] = { 56 + &iommu_pmu_format_attr_group, 57 + &iommu_pmu_events_attr_group, 58 + &iommu_pmu_cpumask_attr_group, 59 + NULL 60 + }; 61 + 62 + static inline struct iommu_pmu *dev_to_iommu_pmu(struct device *dev) 63 + { 64 + /* 65 + * The perf_event creates its own dev for each PMU. 66 + * See pmu_dev_alloc() 67 + */ 68 + return container_of(dev_get_drvdata(dev), struct iommu_pmu, pmu); 69 + } 70 + 71 + #define IOMMU_PMU_ATTR(_name, _format, _filter) \ 72 + PMU_FORMAT_ATTR(_name, _format); \ 73 + \ 74 + static struct attribute *_name##_attr[] = { \ 75 + &format_attr_##_name.attr, \ 76 + NULL \ 77 + }; \ 78 + \ 79 + static umode_t \ 80 + _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ 81 + { \ 82 + struct device *dev = kobj_to_dev(kobj); \ 83 + struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ 84 + \ 85 + if (!iommu_pmu) \ 86 + return 0; \ 87 + return (iommu_pmu->filter & _filter) ? attr->mode : 0; \ 88 + } \ 89 + \ 90 + static struct attribute_group _name = { \ 91 + .name = "format", \ 92 + .attrs = _name##_attr, \ 93 + .is_visible = _name##_is_visible, \ 94 + }; 95 + 96 + IOMMU_PMU_ATTR(filter_requester_id_en, "config1:0", IOMMU_PMU_FILTER_REQUESTER_ID); 97 + IOMMU_PMU_ATTR(filter_domain_en, "config1:1", IOMMU_PMU_FILTER_DOMAIN); 98 + IOMMU_PMU_ATTR(filter_pasid_en, "config1:2", IOMMU_PMU_FILTER_PASID); 99 + IOMMU_PMU_ATTR(filter_ats_en, "config1:3", IOMMU_PMU_FILTER_ATS); 100 + IOMMU_PMU_ATTR(filter_page_table_en, "config1:4", IOMMU_PMU_FILTER_PAGE_TABLE); 101 + IOMMU_PMU_ATTR(filter_requester_id, "config1:16-31", IOMMU_PMU_FILTER_REQUESTER_ID); 102 + IOMMU_PMU_ATTR(filter_domain, "config1:32-47", IOMMU_PMU_FILTER_DOMAIN); 103 + IOMMU_PMU_ATTR(filter_pasid, "config2:0-21", IOMMU_PMU_FILTER_PASID); 104 + IOMMU_PMU_ATTR(filter_ats, "config2:24-28", IOMMU_PMU_FILTER_ATS); 105 + IOMMU_PMU_ATTR(filter_page_table, "config2:32-36", IOMMU_PMU_FILTER_PAGE_TABLE); 106 + 107 + #define iommu_pmu_en_requester_id(e) ((e) & 0x1) 108 + #define iommu_pmu_en_domain(e) (((e) >> 1) & 0x1) 109 + #define iommu_pmu_en_pasid(e) (((e) >> 2) & 0x1) 110 + #define iommu_pmu_en_ats(e) (((e) >> 3) & 0x1) 111 + #define iommu_pmu_en_page_table(e) (((e) >> 4) & 0x1) 112 + #define iommu_pmu_get_requester_id(filter) (((filter) >> 16) & 0xffff) 113 + #define iommu_pmu_get_domain(filter) (((filter) >> 32) & 0xffff) 114 + #define iommu_pmu_get_pasid(filter) ((filter) & 0x3fffff) 115 + #define iommu_pmu_get_ats(filter) (((filter) >> 24) & 0x1f) 116 + #define iommu_pmu_get_page_table(filter) (((filter) >> 32) & 0x1f) 117 + 118 + #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig) \ 119 + { \ 120 + if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) { \ 121 + dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ 122 + IOMMU_PMU_CFG_SIZE + \ 123 + (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ 124 + iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\ 125 + } \ 126 + } 127 + 128 + #define iommu_pmu_clear_filter(_filter, _idx) \ 129 + { \ 130 + if (iommu_pmu->filter & _filter) { \ 131 + dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ 132 + IOMMU_PMU_CFG_SIZE + \ 133 + (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ 134 + 0); \ 135 + } \ 136 + } 137 + 138 + /* 139 + * Define the event attr related functions 140 + * Input: _name: event attr name 141 + * _string: string of the event in sysfs 142 + * _g_idx: event group encoding 143 + * _event: event encoding 144 + */ 145 + #define IOMMU_PMU_EVENT_ATTR(_name, _string, _g_idx, _event) \ 146 + PMU_EVENT_ATTR_STRING(_name, event_attr_##_name, _string) \ 147 + \ 148 + static struct attribute *_name##_attr[] = { \ 149 + &event_attr_##_name.attr.attr, \ 150 + NULL \ 151 + }; \ 152 + \ 153 + static umode_t \ 154 + _name##_is_visible(struct kobject *kobj, struct attribute *attr, int i) \ 155 + { \ 156 + struct device *dev = kobj_to_dev(kobj); \ 157 + struct iommu_pmu *iommu_pmu = dev_to_iommu_pmu(dev); \ 158 + \ 159 + if (!iommu_pmu) \ 160 + return 0; \ 161 + return (iommu_pmu->evcap[_g_idx] & _event) ? attr->mode : 0; \ 162 + } \ 163 + \ 164 + static struct attribute_group _name = { \ 165 + .name = "events", \ 166 + .attrs = _name##_attr, \ 167 + .is_visible = _name##_is_visible, \ 168 + }; 169 + 170 + IOMMU_PMU_EVENT_ATTR(iommu_clocks, "event_group=0x0,event=0x001", 0x0, 0x001) 171 + IOMMU_PMU_EVENT_ATTR(iommu_requests, "event_group=0x0,event=0x002", 0x0, 0x002) 172 + IOMMU_PMU_EVENT_ATTR(pw_occupancy, "event_group=0x0,event=0x004", 0x0, 0x004) 173 + IOMMU_PMU_EVENT_ATTR(ats_blocked, "event_group=0x0,event=0x008", 0x0, 0x008) 174 + IOMMU_PMU_EVENT_ATTR(iommu_mrds, "event_group=0x1,event=0x001", 0x1, 0x001) 175 + IOMMU_PMU_EVENT_ATTR(iommu_mem_blocked, "event_group=0x1,event=0x020", 0x1, 0x020) 176 + IOMMU_PMU_EVENT_ATTR(pg_req_posted, "event_group=0x1,event=0x040", 0x1, 0x040) 177 + IOMMU_PMU_EVENT_ATTR(ctxt_cache_lookup, "event_group=0x2,event=0x001", 0x2, 0x001) 178 + IOMMU_PMU_EVENT_ATTR(ctxt_cache_hit, "event_group=0x2,event=0x002", 0x2, 0x002) 179 + IOMMU_PMU_EVENT_ATTR(pasid_cache_lookup, "event_group=0x2,event=0x004", 0x2, 0x004) 180 + IOMMU_PMU_EVENT_ATTR(pasid_cache_hit, "event_group=0x2,event=0x008", 0x2, 0x008) 181 + IOMMU_PMU_EVENT_ATTR(ss_nonleaf_lookup, "event_group=0x2,event=0x010", 0x2, 0x010) 182 + IOMMU_PMU_EVENT_ATTR(ss_nonleaf_hit, "event_group=0x2,event=0x020", 0x2, 0x020) 183 + IOMMU_PMU_EVENT_ATTR(fs_nonleaf_lookup, "event_group=0x2,event=0x040", 0x2, 0x040) 184 + IOMMU_PMU_EVENT_ATTR(fs_nonleaf_hit, "event_group=0x2,event=0x080", 0x2, 0x080) 185 + IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_lookup, "event_group=0x2,event=0x100", 0x2, 0x100) 186 + IOMMU_PMU_EVENT_ATTR(hpt_nonleaf_hit, "event_group=0x2,event=0x200", 0x2, 0x200) 187 + IOMMU_PMU_EVENT_ATTR(iotlb_lookup, "event_group=0x3,event=0x001", 0x3, 0x001) 188 + IOMMU_PMU_EVENT_ATTR(iotlb_hit, "event_group=0x3,event=0x002", 0x3, 0x002) 189 + IOMMU_PMU_EVENT_ATTR(hpt_leaf_lookup, "event_group=0x3,event=0x004", 0x3, 0x004) 190 + IOMMU_PMU_EVENT_ATTR(hpt_leaf_hit, "event_group=0x3,event=0x008", 0x3, 0x008) 191 + IOMMU_PMU_EVENT_ATTR(int_cache_lookup, "event_group=0x4,event=0x001", 0x4, 0x001) 192 + IOMMU_PMU_EVENT_ATTR(int_cache_hit_nonposted, "event_group=0x4,event=0x002", 0x4, 0x002) 193 + IOMMU_PMU_EVENT_ATTR(int_cache_hit_posted, "event_group=0x4,event=0x004", 0x4, 0x004) 194 + 195 + static const struct attribute_group *iommu_pmu_attr_update[] = { 196 + &filter_requester_id_en, 197 + &filter_domain_en, 198 + &filter_pasid_en, 199 + &filter_ats_en, 200 + &filter_page_table_en, 201 + &filter_requester_id, 202 + &filter_domain, 203 + &filter_pasid, 204 + &filter_ats, 205 + &filter_page_table, 206 + &iommu_clocks, 207 + &iommu_requests, 208 + &pw_occupancy, 209 + &ats_blocked, 210 + &iommu_mrds, 211 + &iommu_mem_blocked, 212 + &pg_req_posted, 213 + &ctxt_cache_lookup, 214 + &ctxt_cache_hit, 215 + &pasid_cache_lookup, 216 + &pasid_cache_hit, 217 + &ss_nonleaf_lookup, 218 + &ss_nonleaf_hit, 219 + &fs_nonleaf_lookup, 220 + &fs_nonleaf_hit, 221 + &hpt_nonleaf_lookup, 222 + &hpt_nonleaf_hit, 223 + &iotlb_lookup, 224 + &iotlb_hit, 225 + &hpt_leaf_lookup, 226 + &hpt_leaf_hit, 227 + &int_cache_lookup, 228 + &int_cache_hit_nonposted, 229 + &int_cache_hit_posted, 230 + NULL 231 + }; 232 + 233 + static inline void __iomem * 234 + iommu_event_base(struct iommu_pmu *iommu_pmu, int idx) 235 + { 236 + return iommu_pmu->cntr_reg + idx * iommu_pmu->cntr_stride; 237 + } 238 + 239 + static inline void __iomem * 240 + iommu_config_base(struct iommu_pmu *iommu_pmu, int idx) 241 + { 242 + return iommu_pmu->cfg_reg + idx * IOMMU_PMU_CFG_OFFSET; 243 + } 244 + 245 + static inline struct iommu_pmu *iommu_event_to_pmu(struct perf_event *event) 246 + { 247 + return container_of(event->pmu, struct iommu_pmu, pmu); 248 + } 249 + 250 + static inline u64 iommu_event_config(struct perf_event *event) 251 + { 252 + u64 config = event->attr.config; 253 + 254 + return (iommu_event_select(config) << IOMMU_EVENT_CFG_ES_SHIFT) | 255 + (iommu_event_group(config) << IOMMU_EVENT_CFG_EGI_SHIFT) | 256 + IOMMU_EVENT_CFG_INT; 257 + } 258 + 259 + static inline bool is_iommu_pmu_event(struct iommu_pmu *iommu_pmu, 260 + struct perf_event *event) 261 + { 262 + return event->pmu == &iommu_pmu->pmu; 263 + } 264 + 265 + static int iommu_pmu_validate_event(struct perf_event *event) 266 + { 267 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 268 + u32 event_group = iommu_event_group(event->attr.config); 269 + 270 + if (event_group >= iommu_pmu->num_eg) 271 + return -EINVAL; 272 + 273 + return 0; 274 + } 275 + 276 + static int iommu_pmu_validate_group(struct perf_event *event) 277 + { 278 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 279 + struct perf_event *sibling; 280 + int nr = 0; 281 + 282 + /* 283 + * All events in a group must be scheduled simultaneously. 284 + * Check whether there is enough counters for all the events. 285 + */ 286 + for_each_sibling_event(sibling, event->group_leader) { 287 + if (!is_iommu_pmu_event(iommu_pmu, sibling) || 288 + sibling->state <= PERF_EVENT_STATE_OFF) 289 + continue; 290 + 291 + if (++nr > iommu_pmu->num_cntr) 292 + return -EINVAL; 293 + } 294 + 295 + return 0; 296 + } 297 + 298 + static int iommu_pmu_event_init(struct perf_event *event) 299 + { 300 + struct hw_perf_event *hwc = &event->hw; 301 + 302 + if (event->attr.type != event->pmu->type) 303 + return -ENOENT; 304 + 305 + /* sampling not supported */ 306 + if (event->attr.sample_period) 307 + return -EINVAL; 308 + 309 + if (event->cpu < 0) 310 + return -EINVAL; 311 + 312 + if (iommu_pmu_validate_event(event)) 313 + return -EINVAL; 314 + 315 + hwc->config = iommu_event_config(event); 316 + 317 + return iommu_pmu_validate_group(event); 318 + } 319 + 320 + static void iommu_pmu_event_update(struct perf_event *event) 321 + { 322 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 323 + struct hw_perf_event *hwc = &event->hw; 324 + u64 prev_count, new_count, delta; 325 + int shift = 64 - iommu_pmu->cntr_width; 326 + 327 + again: 328 + prev_count = local64_read(&hwc->prev_count); 329 + new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); 330 + if (local64_xchg(&hwc->prev_count, new_count) != prev_count) 331 + goto again; 332 + 333 + /* 334 + * The counter width is enumerated. Always shift the counter 335 + * before using it. 336 + */ 337 + delta = (new_count << shift) - (prev_count << shift); 338 + delta >>= shift; 339 + 340 + local64_add(delta, &event->count); 341 + } 342 + 343 + static void iommu_pmu_start(struct perf_event *event, int flags) 344 + { 345 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 346 + struct intel_iommu *iommu = iommu_pmu->iommu; 347 + struct hw_perf_event *hwc = &event->hw; 348 + u64 count; 349 + 350 + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 351 + return; 352 + 353 + if (WARN_ON_ONCE(hwc->idx < 0 || hwc->idx >= IOMMU_PMU_IDX_MAX)) 354 + return; 355 + 356 + if (flags & PERF_EF_RELOAD) 357 + WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 358 + 359 + hwc->state = 0; 360 + 361 + /* Always reprogram the period */ 362 + count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); 363 + local64_set((&hwc->prev_count), count); 364 + 365 + /* 366 + * The error of ecmd will be ignored. 367 + * - The existing perf_event subsystem doesn't handle the error. 368 + * Only IOMMU PMU returns runtime HW error. We don't want to 369 + * change the existing generic interfaces for the specific case. 370 + * - It's a corner case caused by HW, which is very unlikely to 371 + * happen. There is nothing SW can do. 372 + * - The worst case is that the user will get <not count> with 373 + * perf command, which can give the user some hints. 374 + */ 375 + ecmd_submit_sync(iommu, DMA_ECMD_ENABLE, hwc->idx, 0); 376 + 377 + perf_event_update_userpage(event); 378 + } 379 + 380 + static void iommu_pmu_stop(struct perf_event *event, int flags) 381 + { 382 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 383 + struct intel_iommu *iommu = iommu_pmu->iommu; 384 + struct hw_perf_event *hwc = &event->hw; 385 + 386 + if (!(hwc->state & PERF_HES_STOPPED)) { 387 + ecmd_submit_sync(iommu, DMA_ECMD_DISABLE, hwc->idx, 0); 388 + 389 + iommu_pmu_event_update(event); 390 + 391 + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 392 + } 393 + } 394 + 395 + static inline int 396 + iommu_pmu_validate_per_cntr_event(struct iommu_pmu *iommu_pmu, 397 + int idx, struct perf_event *event) 398 + { 399 + u32 event_group = iommu_event_group(event->attr.config); 400 + u32 select = iommu_event_select(event->attr.config); 401 + 402 + if (!(iommu_pmu->cntr_evcap[idx][event_group] & select)) 403 + return -EINVAL; 404 + 405 + return 0; 406 + } 407 + 408 + static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu, 409 + struct perf_event *event) 410 + { 411 + struct hw_perf_event *hwc = &event->hw; 412 + int idx; 413 + 414 + /* 415 + * The counters which support limited events are usually at the end. 416 + * Schedule them first to accommodate more events. 417 + */ 418 + for (idx = iommu_pmu->num_cntr - 1; idx >= 0; idx--) { 419 + if (test_and_set_bit(idx, iommu_pmu->used_mask)) 420 + continue; 421 + /* Check per-counter event capabilities */ 422 + if (!iommu_pmu_validate_per_cntr_event(iommu_pmu, idx, event)) 423 + break; 424 + clear_bit(idx, iommu_pmu->used_mask); 425 + } 426 + if (idx < 0) 427 + return -EINVAL; 428 + 429 + iommu_pmu->event_list[idx] = event; 430 + hwc->idx = idx; 431 + 432 + /* config events */ 433 + dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config); 434 + 435 + iommu_pmu_set_filter(requester_id, event->attr.config1, 436 + IOMMU_PMU_FILTER_REQUESTER_ID, idx, 437 + event->attr.config1); 438 + iommu_pmu_set_filter(domain, event->attr.config1, 439 + IOMMU_PMU_FILTER_DOMAIN, idx, 440 + event->attr.config1); 441 + iommu_pmu_set_filter(pasid, event->attr.config1, 442 + IOMMU_PMU_FILTER_PASID, idx, 443 + event->attr.config1); 444 + iommu_pmu_set_filter(ats, event->attr.config2, 445 + IOMMU_PMU_FILTER_ATS, idx, 446 + event->attr.config1); 447 + iommu_pmu_set_filter(page_table, event->attr.config2, 448 + IOMMU_PMU_FILTER_PAGE_TABLE, idx, 449 + event->attr.config1); 450 + 451 + return 0; 452 + } 453 + 454 + static int iommu_pmu_add(struct perf_event *event, int flags) 455 + { 456 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 457 + struct hw_perf_event *hwc = &event->hw; 458 + int ret; 459 + 460 + ret = iommu_pmu_assign_event(iommu_pmu, event); 461 + if (ret < 0) 462 + return ret; 463 + 464 + hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 465 + 466 + if (flags & PERF_EF_START) 467 + iommu_pmu_start(event, 0); 468 + 469 + return 0; 470 + } 471 + 472 + static void iommu_pmu_del(struct perf_event *event, int flags) 473 + { 474 + struct iommu_pmu *iommu_pmu = iommu_event_to_pmu(event); 475 + int idx = event->hw.idx; 476 + 477 + iommu_pmu_stop(event, PERF_EF_UPDATE); 478 + 479 + iommu_pmu_clear_filter(IOMMU_PMU_FILTER_REQUESTER_ID, idx); 480 + iommu_pmu_clear_filter(IOMMU_PMU_FILTER_DOMAIN, idx); 481 + iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PASID, idx); 482 + iommu_pmu_clear_filter(IOMMU_PMU_FILTER_ATS, idx); 483 + iommu_pmu_clear_filter(IOMMU_PMU_FILTER_PAGE_TABLE, idx); 484 + 485 + iommu_pmu->event_list[idx] = NULL; 486 + event->hw.idx = -1; 487 + clear_bit(idx, iommu_pmu->used_mask); 488 + 489 + perf_event_update_userpage(event); 490 + } 491 + 492 + static void iommu_pmu_enable(struct pmu *pmu) 493 + { 494 + struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); 495 + struct intel_iommu *iommu = iommu_pmu->iommu; 496 + 497 + ecmd_submit_sync(iommu, DMA_ECMD_UNFREEZE, 0, 0); 498 + } 499 + 500 + static void iommu_pmu_disable(struct pmu *pmu) 501 + { 502 + struct iommu_pmu *iommu_pmu = container_of(pmu, struct iommu_pmu, pmu); 503 + struct intel_iommu *iommu = iommu_pmu->iommu; 504 + 505 + ecmd_submit_sync(iommu, DMA_ECMD_FREEZE, 0, 0); 506 + } 507 + 508 + static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu) 509 + { 510 + struct perf_event *event; 511 + u64 status; 512 + int i; 513 + 514 + /* 515 + * Two counters may be overflowed very close. Always check 516 + * whether there are more to handle. 517 + */ 518 + while ((status = dmar_readq(iommu_pmu->overflow))) { 519 + for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) { 520 + /* 521 + * Find the assigned event of the counter. 522 + * Accumulate the value into the event->count. 523 + */ 524 + event = iommu_pmu->event_list[i]; 525 + if (!event) { 526 + pr_warn_once("Cannot find the assigned event for counter %d\n", i); 527 + continue; 528 + } 529 + iommu_pmu_event_update(event); 530 + } 531 + 532 + dmar_writeq(iommu_pmu->overflow, status); 533 + } 534 + } 535 + 536 + static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id) 537 + { 538 + struct intel_iommu *iommu = dev_id; 539 + 540 + if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG)) 541 + return IRQ_NONE; 542 + 543 + iommu_pmu_counter_overflow(iommu->pmu); 544 + 545 + /* Clear the status bit */ 546 + dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS); 547 + 548 + return IRQ_HANDLED; 549 + } 550 + 551 + static int __iommu_pmu_register(struct intel_iommu *iommu) 552 + { 553 + struct iommu_pmu *iommu_pmu = iommu->pmu; 554 + 555 + iommu_pmu->pmu.name = iommu->name; 556 + iommu_pmu->pmu.task_ctx_nr = perf_invalid_context; 557 + iommu_pmu->pmu.event_init = iommu_pmu_event_init; 558 + iommu_pmu->pmu.pmu_enable = iommu_pmu_enable; 559 + iommu_pmu->pmu.pmu_disable = iommu_pmu_disable; 560 + iommu_pmu->pmu.add = iommu_pmu_add; 561 + iommu_pmu->pmu.del = iommu_pmu_del; 562 + iommu_pmu->pmu.start = iommu_pmu_start; 563 + iommu_pmu->pmu.stop = iommu_pmu_stop; 564 + iommu_pmu->pmu.read = iommu_pmu_event_update; 565 + iommu_pmu->pmu.attr_groups = iommu_pmu_attr_groups; 566 + iommu_pmu->pmu.attr_update = iommu_pmu_attr_update; 567 + iommu_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; 568 + iommu_pmu->pmu.module = THIS_MODULE; 569 + 570 + return perf_pmu_register(&iommu_pmu->pmu, iommu_pmu->pmu.name, -1); 571 + } 572 + 573 + static inline void __iomem * 574 + get_perf_reg_address(struct intel_iommu *iommu, u32 offset) 575 + { 576 + u32 off = dmar_readl(iommu->reg + offset); 577 + 578 + return iommu->reg + off; 579 + } 580 + 581 + int alloc_iommu_pmu(struct intel_iommu *iommu) 582 + { 583 + struct iommu_pmu *iommu_pmu; 584 + int i, j, ret; 585 + u64 perfcap; 586 + u32 cap; 587 + 588 + if (!ecap_pms(iommu->ecap)) 589 + return 0; 590 + 591 + /* The IOMMU PMU requires the ECMD support as well */ 592 + if (!cap_ecmds(iommu->cap)) 593 + return -ENODEV; 594 + 595 + perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG); 596 + /* The performance monitoring is not supported. */ 597 + if (!perfcap) 598 + return -ENODEV; 599 + 600 + /* Sanity check for the number of the counters and event groups */ 601 + if (!pcap_num_cntr(perfcap) || !pcap_num_event_group(perfcap)) 602 + return -ENODEV; 603 + 604 + /* The interrupt on overflow is required */ 605 + if (!pcap_interrupt(perfcap)) 606 + return -ENODEV; 607 + 608 + /* Check required Enhanced Command Capability */ 609 + if (!ecmd_has_pmu_essential(iommu)) 610 + return -ENODEV; 611 + 612 + iommu_pmu = kzalloc(sizeof(*iommu_pmu), GFP_KERNEL); 613 + if (!iommu_pmu) 614 + return -ENOMEM; 615 + 616 + iommu_pmu->num_cntr = pcap_num_cntr(perfcap); 617 + if (iommu_pmu->num_cntr > IOMMU_PMU_IDX_MAX) { 618 + pr_warn_once("The number of IOMMU counters %d > max(%d), clipping!", 619 + iommu_pmu->num_cntr, IOMMU_PMU_IDX_MAX); 620 + iommu_pmu->num_cntr = IOMMU_PMU_IDX_MAX; 621 + } 622 + 623 + iommu_pmu->cntr_width = pcap_cntr_width(perfcap); 624 + iommu_pmu->filter = pcap_filters_mask(perfcap); 625 + iommu_pmu->cntr_stride = pcap_cntr_stride(perfcap); 626 + iommu_pmu->num_eg = pcap_num_event_group(perfcap); 627 + 628 + iommu_pmu->evcap = kcalloc(iommu_pmu->num_eg, sizeof(u64), GFP_KERNEL); 629 + if (!iommu_pmu->evcap) { 630 + ret = -ENOMEM; 631 + goto free_pmu; 632 + } 633 + 634 + /* Parse event group capabilities */ 635 + for (i = 0; i < iommu_pmu->num_eg; i++) { 636 + u64 pcap; 637 + 638 + pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG + 639 + i * IOMMU_PMU_CAP_REGS_STEP); 640 + iommu_pmu->evcap[i] = pecap_es(pcap); 641 + } 642 + 643 + iommu_pmu->cntr_evcap = kcalloc(iommu_pmu->num_cntr, sizeof(u32 *), GFP_KERNEL); 644 + if (!iommu_pmu->cntr_evcap) { 645 + ret = -ENOMEM; 646 + goto free_pmu_evcap; 647 + } 648 + for (i = 0; i < iommu_pmu->num_cntr; i++) { 649 + iommu_pmu->cntr_evcap[i] = kcalloc(iommu_pmu->num_eg, sizeof(u32), GFP_KERNEL); 650 + if (!iommu_pmu->cntr_evcap[i]) { 651 + ret = -ENOMEM; 652 + goto free_pmu_cntr_evcap; 653 + } 654 + /* 655 + * Set to the global capabilities, will adjust according 656 + * to per-counter capabilities later. 657 + */ 658 + for (j = 0; j < iommu_pmu->num_eg; j++) 659 + iommu_pmu->cntr_evcap[i][j] = (u32)iommu_pmu->evcap[j]; 660 + } 661 + 662 + iommu_pmu->cfg_reg = get_perf_reg_address(iommu, DMAR_PERFCFGOFF_REG); 663 + iommu_pmu->cntr_reg = get_perf_reg_address(iommu, DMAR_PERFCNTROFF_REG); 664 + iommu_pmu->overflow = get_perf_reg_address(iommu, DMAR_PERFOVFOFF_REG); 665 + 666 + /* 667 + * Check per-counter capabilities. All counters should have the 668 + * same capabilities on Interrupt on Overflow Support and Counter 669 + * Width. 670 + */ 671 + for (i = 0; i < iommu_pmu->num_cntr; i++) { 672 + cap = dmar_readl(iommu_pmu->cfg_reg + 673 + i * IOMMU_PMU_CFG_OFFSET + 674 + IOMMU_PMU_CFG_CNTRCAP_OFFSET); 675 + if (!iommu_cntrcap_pcc(cap)) 676 + continue; 677 + 678 + /* 679 + * It's possible that some counters have a different 680 + * capability because of e.g., HW bug. Check the corner 681 + * case here and simply drop those counters. 682 + */ 683 + if ((iommu_cntrcap_cw(cap) != iommu_pmu->cntr_width) || 684 + !iommu_cntrcap_ios(cap)) { 685 + iommu_pmu->num_cntr = i; 686 + pr_warn("PMU counter capability inconsistent, counter number reduced to %d\n", 687 + iommu_pmu->num_cntr); 688 + } 689 + 690 + /* Clear the pre-defined events group */ 691 + for (j = 0; j < iommu_pmu->num_eg; j++) 692 + iommu_pmu->cntr_evcap[i][j] = 0; 693 + 694 + /* Override with per-counter event capabilities */ 695 + for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) { 696 + cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET + 697 + IOMMU_PMU_CFG_CNTREVCAP_OFFSET + 698 + (j * IOMMU_PMU_OFF_REGS_STEP)); 699 + iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap); 700 + /* 701 + * Some events may only be supported by a specific counter. 702 + * Track them in the evcap as well. 703 + */ 704 + iommu_pmu->evcap[iommu_event_group(cap)] |= iommu_event_select(cap); 705 + } 706 + } 707 + 708 + iommu_pmu->iommu = iommu; 709 + iommu->pmu = iommu_pmu; 710 + 711 + return 0; 712 + 713 + free_pmu_cntr_evcap: 714 + for (i = 0; i < iommu_pmu->num_cntr; i++) 715 + kfree(iommu_pmu->cntr_evcap[i]); 716 + kfree(iommu_pmu->cntr_evcap); 717 + free_pmu_evcap: 718 + kfree(iommu_pmu->evcap); 719 + free_pmu: 720 + kfree(iommu_pmu); 721 + 722 + return ret; 723 + } 724 + 725 + void free_iommu_pmu(struct intel_iommu *iommu) 726 + { 727 + struct iommu_pmu *iommu_pmu = iommu->pmu; 728 + 729 + if (!iommu_pmu) 730 + return; 731 + 732 + if (iommu_pmu->evcap) { 733 + int i; 734 + 735 + for (i = 0; i < iommu_pmu->num_cntr; i++) 736 + kfree(iommu_pmu->cntr_evcap[i]); 737 + kfree(iommu_pmu->cntr_evcap); 738 + } 739 + kfree(iommu_pmu->evcap); 740 + kfree(iommu_pmu); 741 + iommu->pmu = NULL; 742 + } 743 + 744 + static int iommu_pmu_set_interrupt(struct intel_iommu *iommu) 745 + { 746 + struct iommu_pmu *iommu_pmu = iommu->pmu; 747 + int irq, ret; 748 + 749 + irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PERF + iommu->seq_id, iommu->node, iommu); 750 + if (irq <= 0) 751 + return -EINVAL; 752 + 753 + snprintf(iommu_pmu->irq_name, sizeof(iommu_pmu->irq_name), "dmar%d-perf", iommu->seq_id); 754 + 755 + iommu->perf_irq = irq; 756 + ret = request_threaded_irq(irq, NULL, iommu_pmu_irq_handler, 757 + IRQF_ONESHOT, iommu_pmu->irq_name, iommu); 758 + if (ret) { 759 + dmar_free_hwirq(irq); 760 + iommu->perf_irq = 0; 761 + return ret; 762 + } 763 + return 0; 764 + } 765 + 766 + static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu) 767 + { 768 + if (!iommu->perf_irq) 769 + return; 770 + 771 + free_irq(iommu->perf_irq, iommu); 772 + dmar_free_hwirq(iommu->perf_irq); 773 + iommu->perf_irq = 0; 774 + } 775 + 776 + static int iommu_pmu_cpu_online(unsigned int cpu) 777 + { 778 + if (cpumask_empty(&iommu_pmu_cpu_mask)) 779 + cpumask_set_cpu(cpu, &iommu_pmu_cpu_mask); 780 + 781 + return 0; 782 + } 783 + 784 + static int iommu_pmu_cpu_offline(unsigned int cpu) 785 + { 786 + struct dmar_drhd_unit *drhd; 787 + struct intel_iommu *iommu; 788 + int target; 789 + 790 + if (!cpumask_test_and_clear_cpu(cpu, &iommu_pmu_cpu_mask)) 791 + return 0; 792 + 793 + target = cpumask_any_but(cpu_online_mask, cpu); 794 + 795 + if (target < nr_cpu_ids) 796 + cpumask_set_cpu(target, &iommu_pmu_cpu_mask); 797 + else 798 + target = -1; 799 + 800 + rcu_read_lock(); 801 + 802 + for_each_iommu(iommu, drhd) { 803 + if (!iommu->pmu) 804 + continue; 805 + perf_pmu_migrate_context(&iommu->pmu->pmu, cpu, target); 806 + } 807 + rcu_read_unlock(); 808 + 809 + return 0; 810 + } 811 + 812 + static int nr_iommu_pmu; 813 + 814 + static int iommu_pmu_cpuhp_setup(struct iommu_pmu *iommu_pmu) 815 + { 816 + int ret; 817 + 818 + if (nr_iommu_pmu++) 819 + return 0; 820 + 821 + ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE, 822 + "driver/iommu/intel/perfmon:online", 823 + iommu_pmu_cpu_online, 824 + iommu_pmu_cpu_offline); 825 + if (ret) 826 + nr_iommu_pmu = 0; 827 + 828 + return ret; 829 + } 830 + 831 + static void iommu_pmu_cpuhp_free(struct iommu_pmu *iommu_pmu) 832 + { 833 + if (--nr_iommu_pmu) 834 + return; 835 + 836 + cpuhp_remove_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE); 837 + } 838 + 839 + void iommu_pmu_register(struct intel_iommu *iommu) 840 + { 841 + struct iommu_pmu *iommu_pmu = iommu->pmu; 842 + 843 + if (!iommu_pmu) 844 + return; 845 + 846 + if (__iommu_pmu_register(iommu)) 847 + goto err; 848 + 849 + if (iommu_pmu_cpuhp_setup(iommu_pmu)) 850 + goto unregister; 851 + 852 + /* Set interrupt for overflow */ 853 + if (iommu_pmu_set_interrupt(iommu)) 854 + goto cpuhp_free; 855 + 856 + return; 857 + 858 + cpuhp_free: 859 + iommu_pmu_cpuhp_free(iommu_pmu); 860 + unregister: 861 + perf_pmu_unregister(&iommu_pmu->pmu); 862 + err: 863 + pr_err("Failed to register PMU for iommu (seq_id = %d)\n", iommu->seq_id); 864 + free_iommu_pmu(iommu); 865 + } 866 + 867 + void iommu_pmu_unregister(struct intel_iommu *iommu) 868 + { 869 + struct iommu_pmu *iommu_pmu = iommu->pmu; 870 + 871 + if (!iommu_pmu) 872 + return; 873 + 874 + iommu_pmu_unset_interrupt(iommu); 875 + iommu_pmu_cpuhp_free(iommu_pmu); 876 + perf_pmu_unregister(&iommu_pmu->pmu); 877 + }

+64

drivers/iommu/intel/perfmon.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + 3 + /* 4 + * PERFCFGOFF_REG, PERFFRZOFF_REG 5 + * PERFOVFOFF_REG, PERFCNTROFF_REG 6 + */ 7 + #define IOMMU_PMU_NUM_OFF_REGS 4 8 + #define IOMMU_PMU_OFF_REGS_STEP 4 9 + 10 + #define IOMMU_PMU_FILTER_REQUESTER_ID 0x01 11 + #define IOMMU_PMU_FILTER_DOMAIN 0x02 12 + #define IOMMU_PMU_FILTER_PASID 0x04 13 + #define IOMMU_PMU_FILTER_ATS 0x08 14 + #define IOMMU_PMU_FILTER_PAGE_TABLE 0x10 15 + 16 + #define IOMMU_PMU_FILTER_EN BIT(31) 17 + 18 + #define IOMMU_PMU_CFG_OFFSET 0x100 19 + #define IOMMU_PMU_CFG_CNTRCAP_OFFSET 0x80 20 + #define IOMMU_PMU_CFG_CNTREVCAP_OFFSET 0x84 21 + #define IOMMU_PMU_CFG_SIZE 0x8 22 + #define IOMMU_PMU_CFG_FILTERS_OFFSET 0x4 23 + 24 + #define IOMMU_PMU_CAP_REGS_STEP 8 25 + 26 + #define iommu_cntrcap_pcc(p) ((p) & 0x1) 27 + #define iommu_cntrcap_cw(p) (((p) >> 8) & 0xff) 28 + #define iommu_cntrcap_ios(p) (((p) >> 16) & 0x1) 29 + #define iommu_cntrcap_egcnt(p) (((p) >> 28) & 0xf) 30 + 31 + #define IOMMU_EVENT_CFG_EGI_SHIFT 8 32 + #define IOMMU_EVENT_CFG_ES_SHIFT 32 33 + #define IOMMU_EVENT_CFG_INT BIT_ULL(1) 34 + 35 + #define iommu_event_select(p) ((p) & 0xfffffff) 36 + #define iommu_event_group(p) (((p) >> 28) & 0xf) 37 + 38 + #ifdef CONFIG_INTEL_IOMMU_PERF_EVENTS 39 + int alloc_iommu_pmu(struct intel_iommu *iommu); 40 + void free_iommu_pmu(struct intel_iommu *iommu); 41 + void iommu_pmu_register(struct intel_iommu *iommu); 42 + void iommu_pmu_unregister(struct intel_iommu *iommu); 43 + #else 44 + static inline int 45 + alloc_iommu_pmu(struct intel_iommu *iommu) 46 + { 47 + return 0; 48 + } 49 + 50 + static inline void 51 + free_iommu_pmu(struct intel_iommu *iommu) 52 + { 53 + } 54 + 55 + static inline void 56 + iommu_pmu_register(struct intel_iommu *iommu) 57 + { 58 + } 59 + 60 + static inline void 61 + iommu_pmu_unregister(struct intel_iommu *iommu) 62 + { 63 + } 64 + #endif /* CONFIG_INTEL_IOMMU_PERF_EVENTS */

+36 -52

drivers/iommu/intel/svm.c

··· 9 9 #include <linux/sched.h> 10 10 #include <linux/sched/mm.h> 11 11 #include <linux/slab.h> 12 - #include <linux/intel-svm.h> 13 12 #include <linux/rculist.h> 14 13 #include <linux/pci.h> 15 14 #include <linux/pci-ats.h> ··· 78 79 } 79 80 iommu->prq = page_address(pages); 80 81 81 - irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 82 + irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); 82 83 if (irq <= 0) { 83 84 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 84 85 iommu->name); ··· 298 299 return 0; 299 300 } 300 301 301 - static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, 302 - struct device *dev, 303 - struct mm_struct *mm) 302 + static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, 303 + struct mm_struct *mm) 304 304 { 305 305 struct device_domain_info *info = dev_iommu_priv_get(dev); 306 306 struct intel_svm_dev *sdev; ··· 311 313 if (!svm) { 312 314 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 313 315 if (!svm) 314 - return ERR_PTR(-ENOMEM); 316 + return -ENOMEM; 315 317 316 318 svm->pasid = mm->pasid; 317 319 svm->mm = mm; ··· 321 323 ret = mmu_notifier_register(&svm->notifier, mm); 322 324 if (ret) { 323 325 kfree(svm); 324 - return ERR_PTR(ret); 326 + return ret; 325 327 } 326 328 327 329 ret = pasid_private_add(svm->pasid, svm); 328 330 if (ret) { 329 331 mmu_notifier_unregister(&svm->notifier, mm); 330 332 kfree(svm); 331 - return ERR_PTR(ret); 333 + return ret; 332 334 } 333 - } 334 - 335 - /* Find the matching device in svm list */ 336 - sdev = svm_lookup_device_by_dev(svm, dev); 337 - if (sdev) { 338 - sdev->users++; 339 - goto success; 340 335 } 341 336 342 337 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); ··· 342 351 sdev->iommu = iommu; 343 352 sdev->did = FLPT_DEFAULT_DID; 344 353 sdev->sid = PCI_DEVID(info->bus, info->devfn); 345 - sdev->users = 1; 346 - sdev->pasid = svm->pasid; 347 - sdev->sva.dev = dev; 348 354 init_rcu_head(&sdev->rcu); 349 355 if (info->ats_enabled) { 350 - sdev->dev_iotlb = 1; 351 356 sdev->qdep = info->ats_qdep; 352 357 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 353 358 sdev->qdep = 0; ··· 357 370 goto free_sdev; 358 371 359 372 list_add_rcu(&sdev->list, &svm->devs); 360 - success: 361 - return &sdev->sva; 373 + 374 + return 0; 362 375 363 376 free_sdev: 364 377 kfree(sdev); ··· 369 382 kfree(svm); 370 383 } 371 384 372 - return ERR_PTR(ret); 385 + return ret; 373 386 } 374 387 375 388 /* Caller must hold pasid_mutex */ ··· 391 404 mm = svm->mm; 392 405 393 406 if (sdev) { 394 - sdev->users--; 395 - if (!sdev->users) { 396 - list_del_rcu(&sdev->list); 397 - /* Flush the PASID cache and IOTLB for this device. 398 - * Note that we do depend on the hardware *not* using 399 - * the PASID any more. Just as we depend on other 400 - * devices never using PASIDs that they have no right 401 - * to use. We have a *shared* PASID table, because it's 402 - * large and has to be physically contiguous. So it's 403 - * hard to be as defensive as we might like. */ 404 - intel_pasid_tear_down_entry(iommu, dev, 405 - svm->pasid, false); 406 - intel_svm_drain_prq(dev, svm->pasid); 407 - kfree_rcu(sdev, rcu); 407 + list_del_rcu(&sdev->list); 408 + /* 409 + * Flush the PASID cache and IOTLB for this device. 410 + * Note that we do depend on the hardware *not* using 411 + * the PASID any more. Just as we depend on other 412 + * devices never using PASIDs that they have no right 413 + * to use. We have a *shared* PASID table, because it's 414 + * large and has to be physically contiguous. So it's 415 + * hard to be as defensive as we might like. 416 + */ 417 + intel_pasid_tear_down_entry(iommu, dev, svm->pasid, false); 418 + intel_svm_drain_prq(dev, svm->pasid); 419 + kfree_rcu(sdev, rcu); 408 420 409 - if (list_empty(&svm->devs)) { 410 - if (svm->notifier.ops) 411 - mmu_notifier_unregister(&svm->notifier, mm); 412 - pasid_private_remove(svm->pasid); 413 - /* We mandate that no page faults may be outstanding 414 - * for the PASID when intel_svm_unbind_mm() is called. 415 - * If that is not obeyed, subtle errors will happen. 416 - * Let's make them less subtle... */ 417 - memset(svm, 0x6b, sizeof(*svm)); 418 - kfree(svm); 419 - } 421 + if (list_empty(&svm->devs)) { 422 + if (svm->notifier.ops) 423 + mmu_notifier_unregister(&svm->notifier, mm); 424 + pasid_private_remove(svm->pasid); 425 + /* 426 + * We mandate that no page faults may be outstanding 427 + * for the PASID when intel_svm_unbind_mm() is called. 428 + * If that is not obeyed, subtle errors will happen. 429 + * Let's make them less subtle... 430 + */ 431 + memset(svm, 0x6b, sizeof(*svm)); 432 + kfree(svm); 420 433 } 421 434 } 422 435 out: ··· 841 854 struct device_domain_info *info = dev_iommu_priv_get(dev); 842 855 struct intel_iommu *iommu = info->iommu; 843 856 struct mm_struct *mm = domain->mm; 844 - struct iommu_sva *sva; 845 - int ret = 0; 857 + int ret; 846 858 847 859 mutex_lock(&pasid_mutex); 848 - sva = intel_svm_bind_mm(iommu, dev, mm); 849 - if (IS_ERR(sva)) 850 - ret = PTR_ERR(sva); 860 + ret = intel_svm_bind_mm(iommu, dev, mm); 851 861 mutex_unlock(&pasid_mutex); 852 862 853 863 return ret;

-1

drivers/iommu/iommu-traces.c

··· 18 18 19 19 /* iommu_device_event */ 20 20 EXPORT_TRACEPOINT_SYMBOL_GPL(attach_device_to_domain); 21 - EXPORT_TRACEPOINT_SYMBOL_GPL(detach_device_from_domain); 22 21 23 22 /* iommu_map_unmap */ 24 23 EXPORT_TRACEPOINT_SYMBOL_GPL(map);

+85 -86

drivers/iommu/iommu.c

··· 371 371 return ret; 372 372 } 373 373 374 + static bool iommu_is_attach_deferred(struct device *dev) 375 + { 376 + const struct iommu_ops *ops = dev_iommu_ops(dev); 377 + 378 + if (ops->is_attach_deferred) 379 + return ops->is_attach_deferred(dev); 380 + 381 + return false; 382 + } 383 + 384 + static int iommu_group_do_dma_first_attach(struct device *dev, void *data) 385 + { 386 + struct iommu_domain *domain = data; 387 + 388 + lockdep_assert_held(&dev->iommu_group->mutex); 389 + 390 + if (iommu_is_attach_deferred(dev)) { 391 + dev->iommu->attach_deferred = 1; 392 + return 0; 393 + } 394 + 395 + return __iommu_attach_device(domain, dev); 396 + } 397 + 374 398 int iommu_probe_device(struct device *dev) 375 399 { 376 400 const struct iommu_ops *ops; ··· 425 401 * attach the default domain. 426 402 */ 427 403 if (group->default_domain && !group->owner) { 428 - ret = __iommu_attach_device(group->default_domain, dev); 404 + ret = iommu_group_do_dma_first_attach(dev, group->default_domain); 429 405 if (ret) { 430 406 mutex_unlock(&group->mutex); 431 407 iommu_group_put(group); ··· 798 774 799 775 ret = iommu_group_create_file(group, 800 776 &iommu_group_attr_reserved_regions); 801 - if (ret) 777 + if (ret) { 778 + kobject_put(group->devices_kobj); 802 779 return ERR_PTR(ret); 780 + } 803 781 804 782 ret = iommu_group_create_file(group, &iommu_group_attr_type); 805 - if (ret) 783 + if (ret) { 784 + kobject_put(group->devices_kobj); 806 785 return ERR_PTR(ret); 786 + } 807 787 808 788 pr_debug("Allocated group %d\n", group->id); 809 789 ··· 958 930 if (map_size) { 959 931 ret = iommu_map(domain, addr - map_size, 960 932 addr - map_size, map_size, 961 - entry->prot); 933 + entry->prot, GFP_KERNEL); 962 934 if (ret) 963 935 goto out; 964 936 map_size = 0; ··· 973 945 iommu_put_resv_regions(dev, &mappings); 974 946 975 947 return ret; 976 - } 977 - 978 - static bool iommu_is_attach_deferred(struct device *dev) 979 - { 980 - const struct iommu_ops *ops = dev_iommu_ops(dev); 981 - 982 - if (ops->is_attach_deferred) 983 - return ops->is_attach_deferred(dev); 984 - 985 - return false; 986 948 } 987 949 988 950 /** ··· 1027 1009 1028 1010 mutex_lock(&group->mutex); 1029 1011 list_add_tail(&device->list, &group->devices); 1030 - if (group->domain && !iommu_is_attach_deferred(dev)) 1031 - ret = __iommu_attach_device(group->domain, dev); 1012 + if (group->domain) 1013 + ret = iommu_group_do_dma_first_attach(dev, group->domain); 1032 1014 mutex_unlock(&group->mutex); 1033 1015 if (ret) 1034 1016 goto err_put_group; ··· 1794 1776 1795 1777 } 1796 1778 1797 - static int iommu_group_do_dma_attach(struct device *dev, void *data) 1798 - { 1799 - struct iommu_domain *domain = data; 1800 - int ret = 0; 1801 - 1802 - if (!iommu_is_attach_deferred(dev)) 1803 - ret = __iommu_attach_device(domain, dev); 1804 - 1805 - return ret; 1806 - } 1807 - 1808 - static int __iommu_group_dma_attach(struct iommu_group *group) 1779 + static int __iommu_group_dma_first_attach(struct iommu_group *group) 1809 1780 { 1810 1781 return __iommu_group_for_each_dev(group, group->default_domain, 1811 - iommu_group_do_dma_attach); 1782 + iommu_group_do_dma_first_attach); 1812 1783 } 1813 1784 1814 1785 static int iommu_group_do_probe_finalize(struct device *dev, void *data) ··· 1862 1855 1863 1856 iommu_group_create_direct_mappings(group); 1864 1857 1865 - ret = __iommu_group_dma_attach(group); 1858 + ret = __iommu_group_dma_first_attach(group); 1866 1859 1867 1860 mutex_unlock(&group->mutex); 1868 1861 ··· 1994 1987 return -ENODEV; 1995 1988 1996 1989 ret = domain->ops->attach_dev(domain, dev); 1997 - if (!ret) 1998 - trace_attach_device_to_domain(dev); 1999 - return ret; 1990 + if (ret) 1991 + return ret; 1992 + dev->iommu->attach_deferred = 0; 1993 + trace_attach_device_to_domain(dev); 1994 + return 0; 2000 1995 } 2001 1996 2002 1997 /** ··· 2043 2034 2044 2035 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) 2045 2036 { 2046 - if (iommu_is_attach_deferred(dev)) 2037 + if (dev->iommu && dev->iommu->attach_deferred) 2047 2038 return __iommu_attach_device(domain, dev); 2048 2039 2049 2040 return 0; 2050 - } 2051 - 2052 - static void __iommu_detach_device(struct iommu_domain *domain, 2053 - struct device *dev) 2054 - { 2055 - if (iommu_is_attach_deferred(dev)) 2056 - return; 2057 - 2058 - domain->ops->detach_dev(domain, dev); 2059 - trace_detach_device_from_domain(dev); 2060 2041 } 2061 2042 2062 2043 void iommu_detach_device(struct iommu_domain *domain, struct device *dev) ··· 2123 2124 2124 2125 ret = __iommu_group_for_each_dev(group, domain, 2125 2126 iommu_group_do_attach_device); 2126 - if (ret == 0) 2127 + if (ret == 0) { 2127 2128 group->domain = domain; 2129 + } else { 2130 + /* 2131 + * To recover from the case when certain device within the 2132 + * group fails to attach to the new domain, we need force 2133 + * attaching all devices back to the old domain. The old 2134 + * domain is compatible for all devices in the group, 2135 + * hence the iommu driver should always return success. 2136 + */ 2137 + struct iommu_domain *old_domain = group->domain; 2138 + 2139 + group->domain = NULL; 2140 + WARN(__iommu_group_set_domain(group, old_domain), 2141 + "iommu driver failed to attach a compatible domain"); 2142 + } 2128 2143 2129 2144 return ret; 2130 2145 } ··· 2167 2154 } 2168 2155 EXPORT_SYMBOL_GPL(iommu_attach_group); 2169 2156 2170 - static int iommu_group_do_detach_device(struct device *dev, void *data) 2157 + static int iommu_group_do_set_platform_dma(struct device *dev, void *data) 2171 2158 { 2172 - struct iommu_domain *domain = data; 2159 + const struct iommu_ops *ops = dev_iommu_ops(dev); 2173 2160 2174 - __iommu_detach_device(domain, dev); 2161 + if (!WARN_ON(!ops->set_platform_dma_ops)) 2162 + ops->set_platform_dma_ops(dev); 2175 2163 2176 2164 return 0; 2177 2165 } ··· 2186 2172 return 0; 2187 2173 2188 2174 /* 2189 - * New drivers should support default domains and so the detach_dev() op 2190 - * will never be called. Otherwise the NULL domain represents some 2175 + * New drivers should support default domains, so set_platform_dma() 2176 + * op will never be called. Otherwise the NULL domain represents some 2191 2177 * platform specific behavior. 2192 2178 */ 2193 2179 if (!new_domain) { 2194 - if (WARN_ON(!group->domain->ops->detach_dev)) 2195 - return -EINVAL; 2196 - __iommu_group_for_each_dev(group, group->domain, 2197 - iommu_group_do_detach_device); 2180 + __iommu_group_for_each_dev(group, NULL, 2181 + iommu_group_do_set_platform_dma); 2198 2182 group->domain = NULL; 2199 2183 return 0; 2200 2184 } ··· 2372 2360 return ret; 2373 2361 } 2374 2362 2375 - static int _iommu_map(struct iommu_domain *domain, unsigned long iova, 2376 - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2363 + int iommu_map(struct iommu_domain *domain, unsigned long iova, 2364 + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 2377 2365 { 2378 2366 const struct iommu_domain_ops *ops = domain->ops; 2379 2367 int ret; 2368 + 2369 + might_sleep_if(gfpflags_allow_blocking(gfp)); 2370 + 2371 + /* Discourage passing strange GFP flags */ 2372 + if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | 2373 + __GFP_HIGHMEM))) 2374 + return -EINVAL; 2380 2375 2381 2376 ret = __iommu_map(domain, iova, paddr, size, prot, gfp); 2382 2377 if (ret == 0 && ops->iotlb_sync_map) ··· 2391 2372 2392 2373 return ret; 2393 2374 } 2394 - 2395 - int iommu_map(struct iommu_domain *domain, unsigned long iova, 2396 - phys_addr_t paddr, size_t size, int prot) 2397 - { 2398 - might_sleep(); 2399 - return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); 2400 - } 2401 2375 EXPORT_SYMBOL_GPL(iommu_map); 2402 - 2403 - int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 2404 - phys_addr_t paddr, size_t size, int prot) 2405 - { 2406 - return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); 2407 - } 2408 - EXPORT_SYMBOL_GPL(iommu_map_atomic); 2409 2376 2410 2377 static size_t __iommu_unmap_pages(struct iommu_domain *domain, 2411 2378 unsigned long iova, size_t size, ··· 2482 2477 } 2483 2478 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 2484 2479 2485 - static ssize_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2486 - struct scatterlist *sg, unsigned int nents, int prot, 2487 - gfp_t gfp) 2480 + ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2481 + struct scatterlist *sg, unsigned int nents, int prot, 2482 + gfp_t gfp) 2488 2483 { 2489 2484 const struct iommu_domain_ops *ops = domain->ops; 2490 2485 size_t len = 0, mapped = 0; 2491 2486 phys_addr_t start; 2492 2487 unsigned int i = 0; 2493 2488 int ret; 2489 + 2490 + might_sleep_if(gfpflags_allow_blocking(gfp)); 2491 + 2492 + /* Discourage passing strange GFP flags */ 2493 + if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | 2494 + __GFP_HIGHMEM))) 2495 + return -EINVAL; 2494 2496 2495 2497 while (i <= nents) { 2496 2498 phys_addr_t s_phys = sg_phys(sg); ··· 2538 2526 2539 2527 return ret; 2540 2528 } 2541 - 2542 - ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 2543 - struct scatterlist *sg, unsigned int nents, int prot) 2544 - { 2545 - might_sleep(); 2546 - return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL); 2547 - } 2548 2529 EXPORT_SYMBOL_GPL(iommu_map_sg); 2549 - 2550 - ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, 2551 - struct scatterlist *sg, unsigned int nents, int prot) 2552 - { 2553 - return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); 2554 - } 2555 2530 2556 2531 /** 2557 2532 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework

+4 -2

drivers/iommu/iommufd/pages.c

··· 456 456 size % PAGE_SIZE); 457 457 458 458 while (size) { 459 - rc = iommu_map(domain, iova, paddr, PAGE_SIZE, prot); 459 + rc = iommu_map(domain, iova, paddr, PAGE_SIZE, prot, 460 + GFP_KERNEL_ACCOUNT); 460 461 if (rc) 461 462 goto err_unmap; 462 463 iova += PAGE_SIZE; ··· 501 500 else 502 501 rc = iommu_map(domain, iova, 503 502 PFN_PHYS(batch->pfns[cur]) + page_offset, 504 - next_iova - iova, area->iommu_prot); 503 + next_iova - iova, area->iommu_prot, 504 + GFP_KERNEL_ACCOUNT); 505 505 if (rc) 506 506 goto err_unmap; 507 507 iova = next_iova;

-28

drivers/iommu/ipmmu-vmsa.c

··· 299 299 mmu->utlb_ctx[utlb] = domain->context_id; 300 300 } 301 301 302 - /* 303 - * Disable MMU translation for the microTLB. 304 - */ 305 - static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, 306 - unsigned int utlb) 307 - { 308 - struct ipmmu_vmsa_device *mmu = domain->mmu; 309 - 310 - ipmmu_imuctr_write(mmu, utlb, 0); 311 - mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; 312 - } 313 - 314 302 static void ipmmu_tlb_flush_all(void *cookie) 315 303 { 316 304 struct ipmmu_vmsa_domain *domain = cookie; ··· 631 643 return 0; 632 644 } 633 645 634 - static void ipmmu_detach_device(struct iommu_domain *io_domain, 635 - struct device *dev) 636 - { 637 - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 638 - struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 639 - unsigned int i; 640 - 641 - for (i = 0; i < fwspec->num_ids; ++i) 642 - ipmmu_utlb_disable(domain, fwspec->ids[i]); 643 - 644 - /* 645 - * TODO: Optimize by disabling the context when no device is attached. 646 - */ 647 - } 648 - 649 646 static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, 650 647 phys_addr_t paddr, size_t pgsize, size_t pgcount, 651 648 int prot, gfp_t gfp, size_t *mapped) ··· 849 876 .of_xlate = ipmmu_of_xlate, 850 877 .default_domain_ops = &(const struct iommu_domain_ops) { 851 878 .attach_dev = ipmmu_attach_device, 852 - .detach_dev = ipmmu_detach_device, 853 879 .map_pages = ipmmu_map, 854 880 .unmap_pages = ipmmu_unmap, 855 881 .flush_iotlb_all = ipmmu_flush_iotlb_all,

+3 -3

drivers/iommu/msm_iommu.c

··· 443 443 return ret; 444 444 } 445 445 446 - static void msm_iommu_detach_dev(struct iommu_domain *domain, 447 - struct device *dev) 446 + static void msm_iommu_set_platform_dma(struct device *dev) 448 447 { 448 + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 449 449 struct msm_priv *priv = to_msm_priv(domain); 450 450 unsigned long flags; 451 451 struct msm_iommu_dev *iommu; ··· 678 678 .domain_alloc = msm_iommu_domain_alloc, 679 679 .probe_device = msm_iommu_probe_device, 680 680 .device_group = generic_device_group, 681 + .set_platform_dma_ops = msm_iommu_set_platform_dma, 681 682 .pgsize_bitmap = MSM_IOMMU_PGSIZES, 682 683 .of_xlate = qcom_iommu_of_xlate, 683 684 .default_domain_ops = &(const struct iommu_domain_ops) { 684 685 .attach_dev = msm_iommu_attach_dev, 685 - .detach_dev = msm_iommu_detach_dev, 686 686 .map_pages = msm_iommu_map, 687 687 .unmap_pages = msm_iommu_unmap, 688 688 /*

-9

drivers/iommu/mtk_iommu.c

··· 710 710 return ret; 711 711 } 712 712 713 - static void mtk_iommu_detach_device(struct iommu_domain *domain, 714 - struct device *dev) 715 - { 716 - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); 717 - 718 - mtk_iommu_config(data, dev, false, 0); 719 - } 720 - 721 713 static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, 722 714 phys_addr_t paddr, size_t pgsize, size_t pgcount, 723 715 int prot, gfp_t gfp, size_t *mapped) ··· 938 946 .owner = THIS_MODULE, 939 947 .default_domain_ops = &(const struct iommu_domain_ops) { 940 948 .attach_dev = mtk_iommu_attach_device, 941 - .detach_dev = mtk_iommu_detach_device, 942 949 .map_pages = mtk_iommu_map, 943 950 .unmap_pages = mtk_iommu_unmap, 944 951 .flush_iotlb_all = mtk_iommu_flush_iotlb_all,

+2 -2

drivers/iommu/mtk_iommu_v1.c

··· 319 319 return 0; 320 320 } 321 321 322 - static void mtk_iommu_v1_detach_device(struct iommu_domain *domain, struct device *dev) 322 + static void mtk_iommu_v1_set_platform_dma(struct device *dev) 323 323 { 324 324 struct mtk_iommu_v1_data *data = dev_iommu_priv_get(dev); 325 325 ··· 585 585 .def_domain_type = mtk_iommu_v1_def_domain_type, 586 586 .device_group = generic_device_group, 587 587 .pgsize_bitmap = MT2701_IOMMU_PAGE_SIZE, 588 + .set_platform_dma_ops = mtk_iommu_v1_set_platform_dma, 588 589 .owner = THIS_MODULE, 589 590 .default_domain_ops = &(const struct iommu_domain_ops) { 590 591 .attach_dev = mtk_iommu_v1_attach_device, 591 - .detach_dev = mtk_iommu_v1_detach_device, 592 592 .map_pages = mtk_iommu_v1_map, 593 593 .unmap_pages = mtk_iommu_v1_unmap, 594 594 .iova_to_phys = mtk_iommu_v1_iova_to_phys,

+96

drivers/iommu/of_iommu.c

··· 10 10 #include <linux/limits.h> 11 11 #include <linux/module.h> 12 12 #include <linux/of.h> 13 + #include <linux/of_address.h> 13 14 #include <linux/of_iommu.h> 14 15 #include <linux/of_pci.h> 15 16 #include <linux/pci.h> ··· 172 171 173 172 return ops; 174 173 } 174 + 175 + static enum iommu_resv_type __maybe_unused 176 + iommu_resv_region_get_type(struct device *dev, 177 + struct resource *phys, 178 + phys_addr_t start, size_t length) 179 + { 180 + phys_addr_t end = start + length - 1; 181 + 182 + /* 183 + * IOMMU regions without an associated physical region cannot be 184 + * mapped and are simply reservations. 185 + */ 186 + if (phys->start >= phys->end) 187 + return IOMMU_RESV_RESERVED; 188 + 189 + /* may be IOMMU_RESV_DIRECT_RELAXABLE for certain cases */ 190 + if (start == phys->start && end == phys->end) 191 + return IOMMU_RESV_DIRECT; 192 + 193 + dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", &phys, 194 + &start, &end); 195 + return IOMMU_RESV_RESERVED; 196 + } 197 + 198 + /** 199 + * of_iommu_get_resv_regions - reserved region driver helper for device tree 200 + * @dev: device for which to get reserved regions 201 + * @list: reserved region list 202 + * 203 + * IOMMU drivers can use this to implement their .get_resv_regions() callback 204 + * for memory regions attached to a device tree node. See the reserved-memory 205 + * device tree bindings on how to use these: 206 + * 207 + * Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt 208 + */ 209 + void of_iommu_get_resv_regions(struct device *dev, struct list_head *list) 210 + { 211 + #if IS_ENABLED(CONFIG_OF_ADDRESS) 212 + struct of_phandle_iterator it; 213 + int err; 214 + 215 + of_for_each_phandle(&it, err, dev->of_node, "memory-region", NULL, 0) { 216 + const __be32 *maps, *end; 217 + struct resource phys; 218 + int size; 219 + 220 + memset(&phys, 0, sizeof(phys)); 221 + 222 + /* 223 + * The "reg" property is optional and can be omitted by reserved-memory regions 224 + * that represent reservations in the IOVA space, which are regions that should 225 + * not be mapped. 226 + */ 227 + if (of_find_property(it.node, "reg", NULL)) { 228 + err = of_address_to_resource(it.node, 0, &phys); 229 + if (err < 0) { 230 + dev_err(dev, "failed to parse memory region %pOF: %d\n", 231 + it.node, err); 232 + continue; 233 + } 234 + } 235 + 236 + maps = of_get_property(it.node, "iommu-addresses", &size); 237 + if (!maps) 238 + continue; 239 + 240 + end = maps + size / sizeof(__be32); 241 + 242 + while (maps < end) { 243 + struct device_node *np; 244 + u32 phandle; 245 + 246 + phandle = be32_to_cpup(maps++); 247 + np = of_find_node_by_phandle(phandle); 248 + 249 + if (np == dev->of_node) { 250 + int prot = IOMMU_READ | IOMMU_WRITE; 251 + struct iommu_resv_region *region; 252 + enum iommu_resv_type type; 253 + phys_addr_t iova; 254 + size_t length; 255 + 256 + maps = of_translate_dma_region(np, maps, &iova, &length); 257 + type = iommu_resv_region_get_type(dev, &phys, iova, length); 258 + 259 + region = iommu_alloc_resv_region(iova, length, prot, type, 260 + GFP_KERNEL); 261 + if (region) 262 + list_add_tail(&region->list, list); 263 + } 264 + } 265 + } 266 + #endif 267 + } 268 + EXPORT_SYMBOL(of_iommu_get_resv_regions);

+3 -3

drivers/iommu/omap-iommu.c

··· 1556 1556 omap_domain->dev = NULL; 1557 1557 } 1558 1558 1559 - static void omap_iommu_detach_dev(struct iommu_domain *domain, 1560 - struct device *dev) 1559 + static void omap_iommu_set_platform_dma(struct device *dev) 1561 1560 { 1561 + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 1562 1562 struct omap_iommu_domain *omap_domain = to_omap_domain(domain); 1563 1563 1564 1564 spin_lock(&omap_domain->lock); ··· 1737 1737 .probe_device = omap_iommu_probe_device, 1738 1738 .release_device = omap_iommu_release_device, 1739 1739 .device_group = omap_iommu_device_group, 1740 + .set_platform_dma_ops = omap_iommu_set_platform_dma, 1740 1741 .pgsize_bitmap = OMAP_IOMMU_PGSIZES, 1741 1742 .default_domain_ops = &(const struct iommu_domain_ops) { 1742 1743 .attach_dev = omap_iommu_attach_dev, 1743 - .detach_dev = omap_iommu_detach_dev, 1744 1744 .map = omap_iommu_map, 1745 1745 .unmap = omap_iommu_unmap, 1746 1746 .iova_to_phys = omap_iommu_iova_to_phys,

-1

drivers/iommu/rockchip-iommu.c

··· 1192 1192 .of_xlate = rk_iommu_of_xlate, 1193 1193 .default_domain_ops = &(const struct iommu_domain_ops) { 1194 1194 .attach_dev = rk_iommu_attach_device, 1195 - .detach_dev = rk_iommu_detach_device, 1196 1195 .map = rk_iommu_map, 1197 1196 .unmap = rk_iommu_unmap, 1198 1197 .iova_to_phys = rk_iommu_iova_to_phys,

+11 -11

drivers/iommu/s390-iommu.c

··· 52 52 if (!s390_domain) 53 53 return NULL; 54 54 55 - s390_domain->dma_table = dma_alloc_cpu_table(); 55 + s390_domain->dma_table = dma_alloc_cpu_table(GFP_KERNEL); 56 56 if (!s390_domain->dma_table) { 57 57 kfree(s390_domain); 58 58 return NULL; ··· 144 144 return 0; 145 145 } 146 146 147 - static void s390_iommu_detach_device(struct iommu_domain *domain, 148 - struct device *dev) 147 + static void s390_iommu_set_platform_dma(struct device *dev) 149 148 { 150 149 struct zpci_dev *zdev = to_zpci_dev(dev); 151 - 152 - WARN_ON(zdev->s390_domain != to_s390_domain(domain)); 153 150 154 151 __s390_iommu_detach_device(zdev); 155 152 zpci_dma_init_device(zdev); ··· 257 260 258 261 static int s390_iommu_validate_trans(struct s390_domain *s390_domain, 259 262 phys_addr_t pa, dma_addr_t dma_addr, 260 - unsigned long nr_pages, int flags) 263 + unsigned long nr_pages, int flags, 264 + gfp_t gfp) 261 265 { 262 266 phys_addr_t page_addr = pa & PAGE_MASK; 263 267 unsigned long *entry; ··· 266 268 int rc; 267 269 268 270 for (i = 0; i < nr_pages; i++) { 269 - entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr); 271 + entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 272 + gfp); 270 273 if (unlikely(!entry)) { 271 274 rc = -ENOMEM; 272 275 goto undo_cpu_trans; ··· 283 284 while (i-- > 0) { 284 285 dma_addr -= PAGE_SIZE; 285 286 entry = dma_walk_cpu_trans(s390_domain->dma_table, 286 - dma_addr); 287 + dma_addr, gfp); 287 288 if (!entry) 288 289 break; 289 290 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); ··· 300 301 int rc = 0; 301 302 302 303 for (i = 0; i < nr_pages; i++) { 303 - entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr); 304 + entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 305 + GFP_ATOMIC); 304 306 if (unlikely(!entry)) { 305 307 rc = -EINVAL; 306 308 break; ··· 339 339 flags |= ZPCI_TABLE_PROTECTED; 340 340 341 341 rc = s390_iommu_validate_trans(s390_domain, paddr, iova, 342 - pgcount, flags); 342 + pgcount, flags, gfp); 343 343 if (!rc) 344 344 *mapped = size; 345 345 ··· 435 435 .probe_device = s390_iommu_probe_device, 436 436 .release_device = s390_iommu_release_device, 437 437 .device_group = generic_device_group, 438 + .set_platform_dma_ops = s390_iommu_set_platform_dma, 438 439 .pgsize_bitmap = SZ_4K, 439 440 .get_resv_regions = s390_iommu_get_resv_regions, 440 441 .default_domain_ops = &(const struct iommu_domain_ops) { 441 442 .attach_dev = s390_iommu_attach_device, 442 - .detach_dev = s390_iommu_detach_device, 443 443 .map_pages = s390_iommu_map_pages, 444 444 .unmap_pages = s390_iommu_unmap_pages, 445 445 .flush_iotlb_all = s390_iommu_flush_iotlb_all,

-16

drivers/iommu/sprd-iommu.c

··· 255 255 return 0; 256 256 } 257 257 258 - static void sprd_iommu_detach_device(struct iommu_domain *domain, 259 - struct device *dev) 260 - { 261 - struct sprd_iommu_domain *dom = to_sprd_domain(domain); 262 - struct sprd_iommu_device *sdev = dom->sdev; 263 - size_t pgt_size = sprd_iommu_pgt_size(domain); 264 - 265 - if (!sdev) 266 - return; 267 - 268 - dma_free_coherent(sdev->dev, pgt_size, dom->pgt_va, dom->pgt_pa); 269 - sprd_iommu_hw_en(sdev, false); 270 - dom->sdev = NULL; 271 - } 272 - 273 258 static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova, 274 259 phys_addr_t paddr, size_t pgsize, size_t pgcount, 275 260 int prot, gfp_t gfp, size_t *mapped) ··· 399 414 .owner = THIS_MODULE, 400 415 .default_domain_ops = &(const struct iommu_domain_ops) { 401 416 .attach_dev = sprd_iommu_attach_device, 402 - .detach_dev = sprd_iommu_detach_device, 403 417 .map_pages = sprd_iommu_map, 404 418 .unmap_pages = sprd_iommu_unmap, 405 419 .iotlb_sync_map = sprd_iommu_sync_map,

-1

drivers/iommu/sun50i-iommu.c

··· 834 834 .probe_device = sun50i_iommu_probe_device, 835 835 .default_domain_ops = &(const struct iommu_domain_ops) { 836 836 .attach_dev = sun50i_iommu_attach_device, 837 - .detach_dev = sun50i_iommu_detach_device, 838 837 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 839 838 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map, 840 839 .iotlb_sync = sun50i_iommu_iotlb_sync,

+3 -3

drivers/iommu/tegra-gart.c

··· 124 124 return ret; 125 125 } 126 126 127 - static void gart_iommu_detach_dev(struct iommu_domain *domain, 128 - struct device *dev) 127 + static void gart_iommu_set_platform_dma(struct device *dev) 129 128 { 129 + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 130 130 struct gart_device *gart = gart_handle; 131 131 132 132 spin_lock(&gart->dom_lock); ··· 270 270 .domain_alloc = gart_iommu_domain_alloc, 271 271 .probe_device = gart_iommu_probe_device, 272 272 .device_group = generic_device_group, 273 + .set_platform_dma_ops = gart_iommu_set_platform_dma, 273 274 .pgsize_bitmap = GART_IOMMU_PGSIZES, 274 275 .of_xlate = gart_iommu_of_xlate, 275 276 .default_domain_ops = &(const struct iommu_domain_ops) { 276 277 .attach_dev = gart_iommu_attach_dev, 277 - .detach_dev = gart_iommu_detach_dev, 278 278 .map = gart_iommu_map, 279 279 .unmap = gart_iommu_unmap, 280 280 .iova_to_phys = gart_iommu_iova_to_phys,

+3 -2

drivers/iommu/tegra-smmu.c

··· 511 511 return err; 512 512 } 513 513 514 - static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 514 + static void tegra_smmu_set_platform_dma(struct device *dev) 515 515 { 516 + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 516 517 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 517 518 struct tegra_smmu_as *as = to_smmu_as(domain); 518 519 struct tegra_smmu *smmu = as->smmu; ··· 966 965 .domain_alloc = tegra_smmu_domain_alloc, 967 966 .probe_device = tegra_smmu_probe_device, 968 967 .device_group = tegra_smmu_device_group, 968 + .set_platform_dma_ops = tegra_smmu_set_platform_dma, 969 969 .of_xlate = tegra_smmu_of_xlate, 970 970 .pgsize_bitmap = SZ_4K, 971 971 .default_domain_ops = &(const struct iommu_domain_ops) { 972 972 .attach_dev = tegra_smmu_attach_dev, 973 - .detach_dev = tegra_smmu_detach_dev, 974 973 .map = tegra_smmu_map, 975 974 .unmap = tegra_smmu_unmap, 976 975 .iova_to_phys = tegra_smmu_iova_to_phys,

+1 -1

drivers/media/platform/qcom/venus/firmware.c

··· 158 158 core->fw.mapped_mem_size = mem_size; 159 159 160 160 ret = iommu_map(iommu, VENUS_FW_START_ADDR, mem_phys, mem_size, 161 - IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV); 161 + IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV, GFP_KERNEL); 162 162 if (ret) { 163 163 dev_err(dev, "could not map video firmware region\n"); 164 164 return ret;

+4 -2

drivers/net/ipa/ipa_mem.c

··· 472 472 size = PAGE_ALIGN(size + addr - phys); 473 473 iova = phys; /* We just want a direct mapping */ 474 474 475 - ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 475 + ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE, 476 + GFP_KERNEL); 476 477 if (ret) 477 478 return ret; 478 479 ··· 581 580 size = PAGE_ALIGN(size + addr - phys); 582 581 iova = phys; /* We just want a direct mapping */ 583 582 584 - ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 583 + ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE, 584 + GFP_KERNEL); 585 585 if (ret) 586 586 return ret; 587 587

+1 -1

drivers/net/wireless/ath/ath10k/snoc.c

··· 1639 1639 1640 1640 ret = iommu_map(iommu_dom, ar_snoc->fw.fw_start_addr, 1641 1641 ar->msa.paddr, ar->msa.mem_size, 1642 - IOMMU_READ | IOMMU_WRITE); 1642 + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); 1643 1643 if (ret) { 1644 1644 ath10k_err(ar, "failed to map firmware region: %d\n", ret); 1645 1645 goto err_iommu_detach;

+2 -2

drivers/net/wireless/ath/ath11k/ahb.c

··· 1036 1036 1037 1037 ret = iommu_map(iommu_dom, ab_ahb->fw.msa_paddr, 1038 1038 ab_ahb->fw.msa_paddr, ab_ahb->fw.msa_size, 1039 - IOMMU_READ | IOMMU_WRITE); 1039 + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); 1040 1040 if (ret) { 1041 1041 ath11k_err(ab, "failed to map firmware region: %d\n", ret); 1042 1042 goto err_iommu_detach; ··· 1044 1044 1045 1045 ret = iommu_map(iommu_dom, ab_ahb->fw.ce_paddr, 1046 1046 ab_ahb->fw.ce_paddr, ab_ahb->fw.ce_size, 1047 - IOMMU_READ | IOMMU_WRITE); 1047 + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); 1048 1048 if (ret) { 1049 1049 ath11k_err(ab, "failed to map firmware CE region: %d\n", ret); 1050 1050 goto err_iommu_unmap;

+41

drivers/of/address.c

··· 626 626 } 627 627 EXPORT_SYMBOL(of_translate_dma_address); 628 628 629 + /** 630 + * of_translate_dma_region - Translate device tree address and size tuple 631 + * @dev: device tree node for which to translate 632 + * @prop: pointer into array of cells 633 + * @start: return value for the start of the DMA range 634 + * @length: return value for the length of the DMA range 635 + * 636 + * Returns a pointer to the cell immediately following the translated DMA region. 637 + */ 638 + const __be32 *of_translate_dma_region(struct device_node *dev, const __be32 *prop, 639 + phys_addr_t *start, size_t *length) 640 + { 641 + struct device_node *parent; 642 + u64 address, size; 643 + int na, ns; 644 + 645 + parent = __of_get_dma_parent(dev); 646 + if (!parent) 647 + return NULL; 648 + 649 + na = of_bus_n_addr_cells(parent); 650 + ns = of_bus_n_size_cells(parent); 651 + 652 + of_node_put(parent); 653 + 654 + address = of_translate_dma_address(dev, prop); 655 + if (address == OF_BAD_ADDR) 656 + return NULL; 657 + 658 + size = of_read_number(prop + na, ns); 659 + 660 + if (start) 661 + *start = address; 662 + 663 + if (length) 664 + *length = size; 665 + 666 + return prop + na + ns; 667 + } 668 + EXPORT_SYMBOL(of_translate_dma_region); 669 + 629 670 const __be32 *__of_get_address(struct device_node *dev, int index, int bar_no, 630 671 u64 *size, unsigned int *flags) 631 672 {

+3 -2

drivers/remoteproc/remoteproc_core.c

··· 643 643 if (!mapping) 644 644 return -ENOMEM; 645 645 646 - ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len, rsc->flags); 646 + ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len, rsc->flags, 647 + GFP_KERNEL); 647 648 if (ret) { 648 649 dev_err(dev, "failed to map devmem: %d\n", ret); 649 650 goto out; ··· 738 737 } 739 738 740 739 ret = iommu_map(rproc->domain, mem->da, dma, mem->len, 741 - mem->flags); 740 + mem->flags, GFP_KERNEL); 742 741 if (ret) { 743 742 dev_err(dev, "iommu_map failed: %d\n", ret); 744 743 goto free_mapping;

+5 -4

drivers/vfio/vfio_iommu_type1.c

··· 1480 1480 1481 1481 list_for_each_entry(d, &iommu->domain_list, next) { 1482 1482 ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, 1483 - npage << PAGE_SHIFT, prot | IOMMU_CACHE); 1483 + npage << PAGE_SHIFT, prot | IOMMU_CACHE, 1484 + GFP_KERNEL); 1484 1485 if (ret) 1485 1486 goto unwind; 1486 1487 ··· 1778 1777 size = npage << PAGE_SHIFT; 1779 1778 } 1780 1779 1781 - ret = iommu_map(domain->domain, iova, phys, 1782 - size, dma->prot | IOMMU_CACHE); 1780 + ret = iommu_map(domain->domain, iova, phys, size, 1781 + dma->prot | IOMMU_CACHE, GFP_KERNEL); 1783 1782 if (ret) { 1784 1783 if (!dma->iommu_mapped) { 1785 1784 vfio_unpin_pages_remote(dma, iova, ··· 1874 1873 continue; 1875 1874 1876 1875 ret = iommu_map(domain->domain, start, page_to_phys(pages), PAGE_SIZE * 2, 1877 - IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); 1876 + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); 1878 1877 if (!ret) { 1879 1878 size_t unmapped = iommu_unmap(domain->domain, start, PAGE_SIZE); 1880 1879

+1 -1

drivers/vhost/vdpa.c

··· 792 792 r = ops->set_map(vdpa, asid, iotlb); 793 793 } else { 794 794 r = iommu_map(v->domain, iova, pa, size, 795 - perm_to_iommu_flags(perm)); 795 + perm_to_iommu_flags(perm), GFP_KERNEL); 796 796 } 797 797 if (r) { 798 798 vhost_iotlb_del_range(iotlb, iova, iova + size - 1);

+1 -1

include/acpi/actbl1.h

··· 804 804 struct acpi_dmar_hardware_unit { 805 805 struct acpi_dmar_header header; 806 806 u8 flags; 807 - u8 reserved; 807 + u8 size; /* Size of the register set */ 808 808 u16 segment; 809 809 u64 address; /* Register Base Address */ 810 810 };

+1

include/linux/cpuhotplug.h

··· 221 221 CPUHP_AP_PERF_X86_CQM_ONLINE, 222 222 CPUHP_AP_PERF_X86_CSTATE_ONLINE, 223 223 CPUHP_AP_PERF_X86_IDXD_ONLINE, 224 + CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE, 224 225 CPUHP_AP_PERF_S390_CF_ONLINE, 225 226 CPUHP_AP_PERF_S390_SF_ONLINE, 226 227 CPUHP_AP_PERF_ARM_CCI_ONLINE,

+1

include/linux/dmar.h

··· 39 39 struct list_head list; /* list of drhd units */ 40 40 struct acpi_dmar_header *hdr; /* ACPI header */ 41 41 u64 reg_base_addr; /* register base address*/ 42 + unsigned long reg_size; /* size of register set */ 42 43 struct dmar_dev_scope *devices;/* target device array */ 43 44 int devices_cnt; /* target device count */ 44 45 u16 segment; /* PCI domain */

-16

include/linux/intel-svm.h

··· 1 - /* SPDX-License-Identifier: GPL-2.0-only */ 2 - /* 3 - * Copyright © 2015 Intel Corporation. 4 - * 5 - * Authors: David Woodhouse <David.Woodhouse@intel.com> 6 - */ 7 - 8 - #ifndef __INTEL_SVM_H__ 9 - #define __INTEL_SVM_H__ 10 - 11 - /* Page Request Queue depth */ 12 - #define PRQ_ORDER 4 13 - #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 14 - #define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5) 15 - 16 - #endif /* __INTEL_SVM_H__ */

+13 -26

include/linux/iommu.h

··· 228 228 * @release_device: Remove device from iommu driver handling 229 229 * @probe_finalize: Do final setup work after the device is added to an IOMMU 230 230 * group and attached to the groups domain 231 + * @set_platform_dma_ops: Returning control back to the platform DMA ops. This op 232 + * is to support old IOMMU drivers, new drivers should use 233 + * default domains, and the common IOMMU DMA ops. 231 234 * @device_group: find iommu group for a particular device 232 235 * @get_resv_regions: Request list of reserved regions for a device 233 236 * @of_xlate: add OF master IDs to iommu grouping ··· 259 256 struct iommu_device *(*probe_device)(struct device *dev); 260 257 void (*release_device)(struct device *dev); 261 258 void (*probe_finalize)(struct device *dev); 259 + void (*set_platform_dma_ops)(struct device *dev); 262 260 struct iommu_group *(*device_group)(struct device *dev); 263 261 264 262 /* Request/Free a list of reserved regions for a device */ ··· 299 295 * * EBUSY - device is attached to a domain and cannot be changed 300 296 * * ENODEV - device specific errors, not able to be attached 301 297 * * <others> - treated as ENODEV by the caller. Use is discouraged 302 - * @detach_dev: detach an iommu domain from a device 303 298 * @set_dev_pasid: set an iommu domain to a pasid of device 304 299 * @map: map a physically contiguous memory region to an iommu domain 305 300 * @map_pages: map a physically contiguous set of pages of the same size to ··· 319 316 */ 320 317 struct iommu_domain_ops { 321 318 int (*attach_dev)(struct iommu_domain *domain, struct device *dev); 322 - void (*detach_dev)(struct iommu_domain *domain, struct device *dev); 323 319 int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev, 324 320 ioasid_t pasid); 325 321 ··· 403 401 * @iommu_dev: IOMMU device this device is linked to 404 402 * @priv: IOMMU Driver private data 405 403 * @max_pasids: number of PASIDs this device can consume 404 + * @attach_deferred: the dma domain attachment is deferred 406 405 * 407 406 * TODO: migrate other per device data pointers under iommu_dev_data, e.g. 408 407 * struct iommu_group *iommu_group; ··· 416 413 struct iommu_device *iommu_dev; 417 414 void *priv; 418 415 u32 max_pasids; 416 + u32 attach_deferred:1; 419 417 }; 420 418 421 419 int iommu_device_register(struct iommu_device *iommu, ··· 471 467 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); 472 468 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); 473 469 extern int iommu_map(struct iommu_domain *domain, unsigned long iova, 474 - phys_addr_t paddr, size_t size, int prot); 475 - extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 476 - phys_addr_t paddr, size_t size, int prot); 470 + phys_addr_t paddr, size_t size, int prot, gfp_t gfp); 477 471 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, 478 472 size_t size); 479 473 extern size_t iommu_unmap_fast(struct iommu_domain *domain, 480 474 unsigned long iova, size_t size, 481 475 struct iommu_iotlb_gather *iotlb_gather); 482 476 extern ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 483 - struct scatterlist *sg, unsigned int nents, int prot); 484 - extern ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, 485 - unsigned long iova, struct scatterlist *sg, 486 - unsigned int nents, int prot); 477 + struct scatterlist *sg, unsigned int nents, 478 + int prot, gfp_t gfp); 487 479 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 488 480 extern void iommu_set_fault_handler(struct iommu_domain *domain, 489 481 iommu_fault_handler_t handler, void *token); ··· 773 773 } 774 774 775 775 static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, 776 - phys_addr_t paddr, size_t size, int prot) 777 - { 778 - return -ENODEV; 779 - } 780 - 781 - static inline int iommu_map_atomic(struct iommu_domain *domain, 782 - unsigned long iova, phys_addr_t paddr, 783 - size_t size, int prot) 776 + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 784 777 { 785 778 return -ENODEV; 786 779 } ··· 793 800 794 801 static inline ssize_t iommu_map_sg(struct iommu_domain *domain, 795 802 unsigned long iova, struct scatterlist *sg, 796 - unsigned int nents, int prot) 797 - { 798 - return -ENODEV; 799 - } 800 - 801 - static inline ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, 802 - unsigned long iova, struct scatterlist *sg, 803 - unsigned int nents, int prot) 803 + unsigned int nents, int prot, gfp_t gfp) 804 804 { 805 805 return -ENODEV; 806 806 } ··· 1104 1118 static inline size_t iommu_map_sgtable(struct iommu_domain *domain, 1105 1119 unsigned long iova, struct sg_table *sgt, int prot) 1106 1120 { 1107 - return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot); 1121 + return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot, 1122 + GFP_KERNEL); 1108 1123 } 1109 1124 1110 1125 #ifdef CONFIG_IOMMU_DEBUGFS

+2

include/linux/of_address.h

··· 38 38 /* Translate a DMA address from device space to CPU space */ 39 39 extern u64 of_translate_dma_address(struct device_node *dev, 40 40 const __be32 *in_addr); 41 + extern const __be32 *of_translate_dma_region(struct device_node *dev, const __be32 *addr, 42 + phys_addr_t *start, size_t *length); 41 43 42 44 #ifdef CONFIG_OF_ADDRESS 43 45 extern u64 of_translate_address(struct device_node *np, const __be32 *addr);

+8

include/linux/of_iommu.h

··· 12 12 struct device_node *master_np, 13 13 const u32 *id); 14 14 15 + extern void of_iommu_get_resv_regions(struct device *dev, 16 + struct list_head *list); 17 + 15 18 #else 16 19 17 20 static inline const struct iommu_ops *of_iommu_configure(struct device *dev, ··· 22 19 const u32 *id) 23 20 { 24 21 return NULL; 22 + } 23 + 24 + static inline void of_iommu_get_resv_regions(struct device *dev, 25 + struct list_head *list) 26 + { 25 27 } 26 28 27 29 #endif /* CONFIG_OF_IOMMU */

-7

include/trace/events/iommu.h

··· 76 76 TP_ARGS(dev) 77 77 ); 78 78 79 - DEFINE_EVENT(iommu_device_event, detach_device_from_domain, 80 - 81 - TP_PROTO(struct device *dev), 82 - 83 - TP_ARGS(dev) 84 - ); 85 - 86 79 TRACE_EVENT(map, 87 80 88 81 TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size),

Configure Feed

Configure Feed