+6 -1

Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml

reviewed

··· 17 17 The content of the cell is the master ID. 18 18 19 19 compatible: 20 20 - const: allwinner,sun50i-h6-iommu 20 20 + oneOf: 21 21 + - const: allwinner,sun50i-h6-iommu 22 22 + - const: allwinner,sun50i-h616-iommu 23 23 + - items: 24 24 + - const: allwinner,sun55i-a523-iommu 25 25 + - const: allwinner,sun50i-h616-iommu 21 26 22 27 reg: 23 28 maxItems: 1

+4 -2

Documentation/devicetree/bindings/iommu/arm,smmu.yaml

reviewed

··· 86 86 - qcom,qcm2290-smmu-500 87 87 - qcom,sa8775p-smmu-500 88 88 - qcom,sc7280-smmu-500 89 89 + - qcom,sc8180x-smmu-500 89 90 - qcom,sc8280xp-smmu-500 90 91 - qcom,sm6115-smmu-500 91 92 - qcom,sm6125-smmu-500 ··· 96 95 - qcom,sm8450-smmu-500 97 96 - qcom,sm8550-smmu-500 98 97 - qcom,sm8650-smmu-500 98 98 + - qcom,x1e80100-smmu-500 99 99 - const: qcom,adreno-smmu 100 100 - const: qcom,smmu-500 101 101 - const: arm,mmu-500 ··· 417 415 compatible: 418 416 contains: 419 417 enum: 418 418 + - qcom,sc8180x-smmu-500 420 419 - qcom,sm6350-smmu-v2 421 420 - qcom,sm7150-smmu-v2 422 421 - qcom,sm8150-smmu-500 ··· 523 520 - enum: 524 521 - qcom,sm8550-smmu-500 525 522 - qcom,sm8650-smmu-500 523 523 + - qcom,x1e80100-smmu-500 526 524 - const: qcom,adreno-smmu 527 525 - const: qcom,smmu-500 528 526 - const: arm,mmu-500 ··· 554 550 - nvidia,smmu-500 555 551 - qcom,qdu1000-smmu-500 556 552 - qcom,sc7180-smmu-500 557 557 - - qcom,sc8180x-smmu-500 558 553 - qcom,sdm670-smmu-500 559 554 - qcom,sdm845-smmu-500 560 555 - qcom,sdx55-smmu-500 561 556 - qcom,sdx65-smmu-500 562 557 - qcom,sm6350-smmu-500 563 558 - qcom,sm6375-smmu-500 564 564 - - qcom,x1e80100-smmu-500 565 559 then: 566 560 properties: 567 561 clock-names: false

-64

Documentation/devicetree/bindings/iommu/msm,iommu-v0.txt

reviewed

··· 1 1 - * QCOM IOMMU 2 2 - 3 3 - The MSM IOMMU is an implementation compatible with the ARM VMSA short 4 4 - descriptor page tables. It provides address translation for bus masters outside 5 5 - of the CPU, each connected to the IOMMU through a port called micro-TLB. 6 6 - 7 7 - Required Properties: 8 8 - 9 9 - - compatible: Must contain "qcom,apq8064-iommu". 10 10 - - reg: Base address and size of the IOMMU registers. 11 11 - - interrupts: Specifiers for the MMU fault interrupts. For instances that 12 12 - support secure mode two interrupts must be specified, for non-secure and 13 13 - secure mode, in that order. For instances that don't support secure mode a 14 14 - single interrupt must be specified. 15 15 - - #iommu-cells: The number of cells needed to specify the stream id. This 16 16 - is always 1. 17 17 - - qcom,ncb: The total number of context banks in the IOMMU. 18 18 - - clocks : List of clocks to be used during SMMU register access. See 19 19 - Documentation/devicetree/bindings/clock/clock-bindings.txt 20 20 - for information about the format. For each clock specified 21 21 - here, there must be a corresponding entry in clock-names 22 22 - (see below). 23 23 - 24 24 - - clock-names : List of clock names corresponding to the clocks specified in 25 25 - the "clocks" property (above). 26 26 - Should be "smmu_pclk" for specifying the interface clock 27 27 - required for iommu's register accesses. 28 28 - Should be "smmu_clk" for specifying the functional clock 29 29 - required by iommu for bus accesses. 30 30 - 31 31 - Each bus master connected to an IOMMU must reference the IOMMU in its device 32 32 - node with the following property: 33 33 - 34 34 - - iommus: A reference to the IOMMU in multiple cells. The first cell is a 35 35 - phandle to the IOMMU and the second cell is the stream id. 36 36 - A single master device can be connected to more than one iommu 37 37 - and multiple contexts in each of the iommu. So multiple entries 38 38 - are required to list all the iommus and the stream ids that the 39 39 - master is connected to. 40 40 - 41 41 - Example: mdp iommu and its bus master 42 42 - 43 43 - mdp_port0: iommu@7500000 { 44 44 - compatible = "qcom,apq8064-iommu"; 45 45 - #iommu-cells = <1>; 46 46 - clock-names = 47 47 - "smmu_pclk", 48 48 - "smmu_clk"; 49 49 - clocks = 50 50 - <&mmcc SMMU_AHB_CLK>, 51 51 - <&mmcc MDP_AXI_CLK>; 52 52 - reg = <0x07500000 0x100000>; 53 53 - interrupts = 54 54 - <GIC_SPI 63 0>, 55 55 - <GIC_SPI 64 0>; 56 56 - qcom,ncb = <2>; 57 57 - }; 58 58 - 59 59 - mdp: qcom,mdp@5100000 { 60 60 - compatible = "qcom,mdp"; 61 61 - ... 62 62 - iommus = <&mdp_port0 0 63 63 - &mdp_port0 2>; 64 64 - };

+78

Documentation/devicetree/bindings/iommu/qcom,apq8064-iommu.yaml

reviewed

··· 1 1 + # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 2 + %YAML 1.2 3 3 + --- 4 4 + 5 5 + $id: http://devicetree.org/schemas/iommu/qcom,apq8064-iommu.yaml# 6 6 + $schema: http://devicetree.org/meta-schemas/core.yaml# 7 7 + 8 8 + title: Qualcomm APQ8064 IOMMU 9 9 + 10 10 + maintainers: 11 11 + - David Heidelberg <david@ixit.cz> 12 12 + 13 13 + description: 14 14 + The MSM IOMMU is an implementation compatible with the ARM VMSA short 15 15 + descriptor page tables. It provides address translation for bus masters 16 16 + outside of the CPU, each connected to the IOMMU through a port called micro-TLB. 17 17 + 18 18 + properties: 19 19 + compatible: 20 20 + const: qcom,apq8064-iommu 21 21 + 22 22 + clocks: 23 23 + items: 24 24 + - description: interface clock for register accesses 25 25 + - description: functional clock for bus accesses 26 26 + 27 27 + clock-names: 28 28 + items: 29 29 + - const: smmu_pclk 30 30 + - const: iommu_clk 31 31 + 32 32 + reg: 33 33 + maxItems: 1 34 34 + 35 35 + interrupts: 36 36 + description: Specifiers for the MMU fault interrupts. 37 37 + minItems: 1 38 38 + items: 39 39 + - description: non-secure mode interrupt 40 40 + - description: secure mode interrupt (for instances which supports it) 41 41 + 42 42 + "#iommu-cells": 43 43 + const: 1 44 44 + description: Each IOMMU specifier describes a single Stream ID. 45 45 + 46 46 + qcom,ncb: 47 47 + $ref: /schemas/types.yaml#/definitions/uint32 48 48 + description: The total number of context banks in the IOMMU. 49 49 + minimum: 1 50 50 + maximum: 4 51 51 + 52 52 + required: 53 53 + - reg 54 54 + - interrupts 55 55 + - clocks 56 56 + - clock-names 57 57 + - qcom,ncb 58 58 + 59 59 + additionalProperties: false 60 60 + 61 61 + examples: 62 62 + - | 63 63 + #include <dt-bindings/clock/qcom,mmcc-msm8960.h> 64 64 + #include <dt-bindings/interrupt-controller/irq.h> 65 65 + #include <dt-bindings/interrupt-controller/arm-gic.h> 66 66 + 67 67 + iommu@7500000 { 68 68 + compatible = "qcom,apq8064-iommu"; 69 69 + reg = <0x07500000 0x100000>; 70 70 + interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>, 71 71 + <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>; 72 72 + clocks = <&clk SMMU_AHB_CLK>, 73 73 + <&clk MDP_AXI_CLK>; 74 74 + clock-names = "smmu_pclk", 75 75 + "iommu_clk"; 76 76 + #iommu-cells = <1>; 77 77 + qcom,ncb = <2>; 78 78 + };

+1

Documentation/devicetree/bindings/iommu/qcom,iommu.yaml

reviewed

··· 25 25 - const: qcom,msm-iommu-v1 26 26 - items: 27 27 - enum: 28 28 + - qcom,msm8953-iommu 28 29 - qcom,msm8976-iommu 29 30 - const: qcom,msm-iommu-v2 30 31

+6

Documentation/devicetree/bindings/pci/host-generic-pci.yaml

reviewed

··· 110 110 iommu-map-mask: true 111 111 msi-parent: true 112 112 113 113 + ats-supported: 114 114 + description: 115 115 + Indicates that a PCIe host controller supports ATS, and can handle Memory 116 116 + Requests with Address Type (AT). 117 117 + type: boolean 118 118 + 113 119 required: 114 120 - compatible 115 121 - reg

-1

Documentation/userspace-api/index.rst

reviewed

··· 45 45 accelerators/ocxl 46 46 dma-buf-alloc-exchange 47 47 gpio/index 48 48 - iommu 49 48 iommufd 50 49 media/index 51 50 dcdbas

-209

Documentation/userspace-api/iommu.rst

reviewed

··· 1 1 - .. SPDX-License-Identifier: GPL-2.0 2 2 - .. iommu: 3 3 - 4 4 - ===================================== 5 5 - IOMMU Userspace API 6 6 - ===================================== 7 7 - 8 8 - IOMMU UAPI is used for virtualization cases where communications are 9 9 - needed between physical and virtual IOMMU drivers. For baremetal 10 10 - usage, the IOMMU is a system device which does not need to communicate 11 11 - with userspace directly. 12 12 - 13 13 - The primary use cases are guest Shared Virtual Address (SVA) and 14 14 - guest IO virtual address (IOVA), wherein the vIOMMU implementation 15 15 - relies on the physical IOMMU and for this reason requires interactions 16 16 - with the host driver. 17 17 - 18 18 - .. contents:: :local: 19 19 - 20 20 - Functionalities 21 21 - =============== 22 22 - Communications of user and kernel involve both directions. The 23 23 - supported user-kernel APIs are as follows: 24 24 - 25 25 - 1. Bind/Unbind guest PASID (e.g. Intel VT-d) 26 26 - 2. Bind/Unbind guest PASID table (e.g. ARM SMMU) 27 27 - 3. Invalidate IOMMU caches upon guest requests 28 28 - 4. Report errors to the guest and serve page requests 29 29 - 30 30 - Requirements 31 31 - ============ 32 32 - The IOMMU UAPIs are generic and extensible to meet the following 33 33 - requirements: 34 34 - 35 35 - 1. Emulated and para-virtualised vIOMMUs 36 36 - 2. Multiple vendors (Intel VT-d, ARM SMMU, etc.) 37 37 - 3. Extensions to the UAPI shall not break existing userspace 38 38 - 39 39 - Interfaces 40 40 - ========== 41 41 - Although the data structures defined in IOMMU UAPI are self-contained, 42 42 - there are no user API functions introduced. Instead, IOMMU UAPI is 43 43 - designed to work with existing user driver frameworks such as VFIO. 44 44 - 45 45 - Extension Rules & Precautions 46 46 - ----------------------------- 47 47 - When IOMMU UAPI gets extended, the data structures can *only* be 48 48 - modified in two ways: 49 49 - 50 50 - 1. Adding new fields by re-purposing the padding[] field. No size change. 51 51 - 2. Adding new union members at the end. May increase the structure sizes. 52 52 - 53 53 - No new fields can be added *after* the variable sized union in that it 54 54 - will break backward compatibility when offset moves. A new flag must 55 55 - be introduced whenever a change affects the structure using either 56 56 - method. The IOMMU driver processes the data based on flags which 57 57 - ensures backward compatibility. 58 58 - 59 59 - Version field is only reserved for the unlikely event of UAPI upgrade 60 60 - at its entirety. 61 61 - 62 62 - It's *always* the caller's responsibility to indicate the size of the 63 63 - structure passed by setting argsz appropriately. 64 64 - Though at the same time, argsz is user provided data which is not 65 65 - trusted. The argsz field allows the user app to indicate how much data 66 66 - it is providing; it's still the kernel's responsibility to validate 67 67 - whether it's correct and sufficient for the requested operation. 68 68 - 69 69 - Compatibility Checking 70 70 - ---------------------- 71 71 - When IOMMU UAPI extension results in some structure size increase, 72 72 - IOMMU UAPI code shall handle the following cases: 73 73 - 74 74 - 1. User and kernel has exact size match 75 75 - 2. An older user with older kernel header (smaller UAPI size) running on a 76 76 - newer kernel (larger UAPI size) 77 77 - 3. A newer user with newer kernel header (larger UAPI size) running 78 78 - on an older kernel. 79 79 - 4. A malicious/misbehaving user passing illegal/invalid size but within 80 80 - range. The data may contain garbage. 81 81 - 82 82 - Feature Checking 83 83 - ---------------- 84 84 - While launching a guest with vIOMMU, it is strongly advised to check 85 85 - the compatibility upfront, as some subsequent errors happening during 86 86 - vIOMMU operation, such as cache invalidation failures cannot be nicely 87 87 - escalated to the guest due to IOMMU specifications. This can lead to 88 88 - catastrophic failures for the users. 89 89 - 90 90 - User applications such as QEMU are expected to import kernel UAPI 91 91 - headers. Backward compatibility is supported per feature flags. 92 92 - For example, an older QEMU (with older kernel header) can run on newer 93 93 - kernel. Newer QEMU (with new kernel header) may refuse to initialize 94 94 - on an older kernel if new feature flags are not supported by older 95 95 - kernel. Simply recompiling existing code with newer kernel header should 96 96 - not be an issue in that only existing flags are used. 97 97 - 98 98 - IOMMU vendor driver should report the below features to IOMMU UAPI 99 99 - consumers (e.g. via VFIO). 100 100 - 101 101 - 1. IOMMU_NESTING_FEAT_SYSWIDE_PASID 102 102 - 2. IOMMU_NESTING_FEAT_BIND_PGTBL 103 103 - 3. IOMMU_NESTING_FEAT_BIND_PASID_TABLE 104 104 - 4. IOMMU_NESTING_FEAT_CACHE_INVLD 105 105 - 5. IOMMU_NESTING_FEAT_PAGE_REQUEST 106 106 - 107 107 - Take VFIO as example, upon request from VFIO userspace (e.g. QEMU), 108 108 - VFIO kernel code shall query IOMMU vendor driver for the support of 109 109 - the above features. Query result can then be reported back to the 110 110 - userspace caller. Details can be found in 111 111 - Documentation/driver-api/vfio.rst. 112 112 - 113 113 - 114 114 - Data Passing Example with VFIO 115 115 - ------------------------------ 116 116 - As the ubiquitous userspace driver framework, VFIO is already IOMMU 117 117 - aware and shares many key concepts such as device model, group, and 118 118 - protection domain. Other user driver frameworks can also be extended 119 119 - to support IOMMU UAPI but it is outside the scope of this document. 120 120 - 121 121 - In this tight-knit VFIO-IOMMU interface, the ultimate consumer of the 122 122 - IOMMU UAPI data is the host IOMMU driver. VFIO facilitates user-kernel 123 123 - transport, capability checking, security, and life cycle management of 124 124 - process address space ID (PASID). 125 125 - 126 126 - VFIO layer conveys the data structures down to the IOMMU driver. It 127 127 - follows the pattern below:: 128 128 - 129 129 - struct { 130 130 - __u32 argsz; 131 131 - __u32 flags; 132 132 - __u8 data[]; 133 133 - }; 134 134 - 135 135 - Here data[] contains the IOMMU UAPI data structures. VFIO has the 136 136 - freedom to bundle the data as well as parse data size based on its own flags. 137 137 - 138 138 - In order to determine the size and feature set of the user data, argsz 139 139 - and flags (or the equivalent) are also embedded in the IOMMU UAPI data 140 140 - structures. 141 141 - 142 142 - A "__u32 argsz" field is *always* at the beginning of each structure. 143 143 - 144 144 - For example: 145 145 - :: 146 146 - 147 147 - struct iommu_cache_invalidate_info { 148 148 - __u32 argsz; 149 149 - #define IOMMU_CACHE_INVALIDATE_INFO_VERSION_1 1 150 150 - __u32 version; 151 151 - /* IOMMU paging structure cache */ 152 152 - #define IOMMU_CACHE_INV_TYPE_IOTLB (1 << 0) /* IOMMU IOTLB */ 153 153 - #define IOMMU_CACHE_INV_TYPE_DEV_IOTLB (1 << 1) /* Device IOTLB */ 154 154 - #define IOMMU_CACHE_INV_TYPE_PASID (1 << 2) /* PASID cache */ 155 155 - #define IOMMU_CACHE_INV_TYPE_NR (3) 156 156 - __u8 cache; 157 157 - __u8 granularity; 158 158 - __u8 padding[6]; 159 159 - union { 160 160 - struct iommu_inv_pasid_info pasid_info; 161 161 - struct iommu_inv_addr_info addr_info; 162 162 - } granu; 163 163 - }; 164 164 - 165 165 - VFIO is responsible for checking its own argsz and flags. It then 166 166 - invokes appropriate IOMMU UAPI functions. The user pointers are passed 167 167 - to the IOMMU layer for further processing. The responsibilities are 168 168 - divided as follows: 169 169 - 170 170 - - Generic IOMMU layer checks argsz range based on UAPI data in the 171 171 - current kernel version. 172 172 - 173 173 - - Generic IOMMU layer checks content of the UAPI data for non-zero 174 174 - reserved bits in flags, padding fields, and unsupported version. 175 175 - This is to ensure not breaking userspace in the future when these 176 176 - fields or flags are used. 177 177 - 178 178 - - Vendor IOMMU driver checks argsz based on vendor flags. UAPI data 179 179 - is consumed based on flags. Vendor driver has access to 180 180 - unadulterated argsz value in case of vendor specific future 181 181 - extensions. Currently, it does not perform the copy_from_user() 182 182 - itself. A __user pointer can be provided in some future scenarios 183 183 - where there's vendor data outside of the structure definition. 184 184 - 185 185 - IOMMU code treats UAPI data in two categories: 186 186 - 187 187 - - structure contains vendor data 188 188 - (Example: iommu_uapi_cache_invalidate()) 189 189 - 190 190 - - structure contains only generic data 191 191 - (Example: iommu_uapi_sva_bind_gpasid()) 192 192 - 193 193 - 194 194 - 195 195 - Sharing UAPI with in-kernel users 196 196 - --------------------------------- 197 197 - For UAPIs that are shared with in-kernel users, a wrapper function is 198 198 - provided to distinguish the callers. For example, 199 199 - 200 200 - Userspace caller :: 201 201 - 202 202 - int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, 203 203 - struct device *dev, 204 204 - void __user *udata) 205 205 - 206 206 - In-kernel caller :: 207 207 - 208 208 - int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 209 209 - struct device *dev, ioasid_t ioasid);

-1

MAINTAINERS

reviewed

··· 11685 11685 S: Maintained 11686 11686 T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git 11687 11687 F: Documentation/devicetree/bindings/iommu/ 11688 11688 - F: Documentation/userspace-api/iommu.rst 11689 11688 F: drivers/iommu/ 11690 11689 F: include/linux/iommu.h 11691 11690 F: include/linux/iova.h

+1

arch/arm64/boot/dts/arm/fvp-base-revc.dts

reviewed

··· 243 243 iommu-map = <0x0 &smmu 0x0 0x10000>; 244 244 245 245 dma-coherent; 246 246 + ats-supported; 246 247 }; 247 248 248 249 smmu: iommu@2b400000 {

+5 -14

drivers/acpi/arm64/iort.c

reviewed

··· 1221 1221 static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node, 1222 1222 u32 streamid) 1223 1223 { 1224 1224 - const struct iommu_ops *ops; 1225 1224 struct fwnode_handle *iort_fwnode; 1226 1225 1227 1227 - if (!node) 1226 1226 + /* If there's no SMMU driver at all, give up now */ 1227 1227 + if (!node || !iort_iommu_driver_enabled(node->type)) 1228 1228 return -ENODEV; 1229 1229 1230 1230 iort_fwnode = iort_get_fwnode(node); ··· 1232 1232 return -ENODEV; 1233 1233 1234 1234 /* 1235 1235 - * If the ops look-up fails, this means that either 1236 1236 - * the SMMU drivers have not been probed yet or that 1237 1237 - * the SMMU drivers are not built in the kernel; 1238 1238 - * Depending on whether the SMMU drivers are built-in 1239 1239 - * in the kernel or not, defer the IOMMU configuration 1240 1240 - * or just abort it. 1235 1235 + * If the SMMU drivers are enabled but not loaded/probed 1236 1236 + * yet, this will defer. 1241 1237 */ 1242 1242 - ops = iommu_ops_from_fwnode(iort_fwnode); 1243 1243 - if (!ops) 1244 1244 - return iort_iommu_driver_enabled(node->type) ? 1245 1245 - -EPROBE_DEFER : -ENODEV; 1246 1246 - 1247 1247 - return acpi_iommu_fwspec_init(dev, streamid, iort_fwnode, ops); 1238 1238 + return acpi_iommu_fwspec_init(dev, streamid, iort_fwnode); 1248 1239 } 1249 1240 1250 1241 struct iort_pci_alias_info {

+8 -28

drivers/acpi/scan.c

reviewed

··· 1606 1606 1607 1607 #ifdef CONFIG_IOMMU_API 1608 1608 int acpi_iommu_fwspec_init(struct device *dev, u32 id, 1609 1609 - struct fwnode_handle *fwnode, 1610 1610 - const struct iommu_ops *ops) 1609 1609 + struct fwnode_handle *fwnode) 1611 1610 { 1612 1611 int ret; 1613 1612 1614 1614 - ret = iommu_fwspec_init(dev, fwnode, ops); 1613 1613 + ret = iommu_fwspec_init(dev, fwnode); 1615 1614 if (ret) 1616 1615 return ret; 1617 1616 1618 1617 return iommu_fwspec_add_ids(dev, &id, 1); 1619 1618 } 1620 1619 1621 1621 - static inline const struct iommu_ops *acpi_iommu_fwspec_ops(struct device *dev) 1622 1622 - { 1623 1623 - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1624 1624 - 1625 1625 - return fwspec ? fwspec->ops : NULL; 1626 1626 - } 1627 1627 - 1628 1620 static int acpi_iommu_configure_id(struct device *dev, const u32 *id_in) 1629 1621 { 1630 1622 int err; 1631 1631 - const struct iommu_ops *ops; 1632 1623 1633 1624 /* Serialise to make dev->iommu stable under our potential fwspec */ 1634 1625 mutex_lock(&iommu_probe_device_lock); 1635 1635 - /* 1636 1636 - * If we already translated the fwspec there is nothing left to do, 1637 1637 - * return the iommu_ops. 1638 1638 - */ 1639 1639 - ops = acpi_iommu_fwspec_ops(dev); 1640 1640 - if (ops) { 1626 1626 + /* If we already translated the fwspec there is nothing left to do */ 1627 1627 + if (dev_iommu_fwspec_get(dev)) { 1641 1628 mutex_unlock(&iommu_probe_device_lock); 1642 1629 return 0; 1643 1630 } ··· 1641 1654 if (!err && dev->bus) 1642 1655 err = iommu_probe_device(dev); 1643 1656 1644 1644 - if (err == -EPROBE_DEFER) 1645 1645 - return err; 1646 1646 - if (err) { 1647 1647 - dev_dbg(dev, "Adding to IOMMU failed: %d\n", err); 1648 1648 - return err; 1649 1649 - } 1650 1650 - if (!acpi_iommu_fwspec_ops(dev)) 1651 1651 - return -ENODEV; 1652 1652 - return 0; 1657 1657 + return err; 1653 1658 } 1654 1659 1655 1660 #else /* !CONFIG_IOMMU_API */ 1656 1661 1657 1662 int acpi_iommu_fwspec_init(struct device *dev, u32 id, 1658 1658 - struct fwnode_handle *fwnode, 1659 1659 - const struct iommu_ops *ops) 1663 1663 + struct fwnode_handle *fwnode) 1660 1664 { 1661 1665 return -ENODEV; 1662 1666 } ··· 1681 1703 ret = acpi_iommu_configure_id(dev, input_id); 1682 1704 if (ret == -EPROBE_DEFER) 1683 1705 return -EPROBE_DEFER; 1706 1706 + if (ret) 1707 1707 + dev_dbg(dev, "Adding to IOMMU failed: %d\n", ret); 1684 1708 1685 1709 arch_setup_dma_ops(dev, attr == DEV_DMA_COHERENT); 1686 1710

+2 -9

drivers/acpi/viot.c

reviewed

··· 307 307 static int viot_dev_iommu_init(struct device *dev, struct viot_iommu *viommu, 308 308 u32 epid) 309 309 { 310 310 - const struct iommu_ops *ops; 311 311 - 312 312 - if (!viommu) 310 310 + if (!viommu || !IS_ENABLED(CONFIG_VIRTIO_IOMMU)) 313 311 return -ENODEV; 314 312 315 313 /* We're not translating ourself */ 316 314 if (device_match_fwnode(dev, viommu->fwnode)) 317 315 return -EINVAL; 318 316 319 319 - ops = iommu_ops_from_fwnode(viommu->fwnode); 320 320 - if (!ops) 321 321 - return IS_ENABLED(CONFIG_VIRTIO_IOMMU) ? 322 322 - -EPROBE_DEFER : -ENODEV; 323 323 - 324 324 - return acpi_iommu_fwspec_init(dev, epid, viommu->fwnode, ops); 317 317 + return acpi_iommu_fwspec_init(dev, epid, viommu->fwnode); 325 318 } 326 319 327 320 static int viot_pci_dev_iommu_init(struct pci_dev *pdev, u16 dev_id, void *data)

+5 -2

drivers/gpu/drm/msm/msm_iommu.c

reviewed

··· 407 407 struct msm_iommu *iommu; 408 408 int ret; 409 409 410 410 - domain = iommu_domain_alloc(dev->bus); 411 411 - if (!domain) 410 410 + if (!device_iommu_mapped(dev)) 412 411 return NULL; 412 412 + 413 413 + domain = iommu_paging_domain_alloc(dev); 414 414 + if (IS_ERR(domain)) 415 415 + return ERR_CAST(domain); 413 416 414 417 iommu_set_pgtable_quirks(domain, quirks); 415 418

+3 -3

drivers/infiniband/hw/usnic/usnic_uiom.c

reviewed

··· 443 443 if (!pd) 444 444 return ERR_PTR(-ENOMEM); 445 445 446 446 - pd->domain = domain = iommu_domain_alloc(dev->bus); 447 447 - if (!domain) { 446 446 + pd->domain = domain = iommu_paging_domain_alloc(dev); 447 447 + if (IS_ERR(domain)) { 448 448 usnic_err("Failed to allocate IOMMU domain"); 449 449 kfree(pd); 450 450 - return ERR_PTR(-ENOMEM); 450 450 + return ERR_CAST(domain); 451 451 } 452 452 453 453 iommu_set_fault_handler(pd->domain, usnic_uiom_dma_fault, NULL);

+1

drivers/iommu/Kconfig

reviewed

··· 394 394 select IOMMU_API 395 395 select IOMMU_IO_PGTABLE_LPAE 396 396 select GENERIC_MSI_IRQ 397 397 + select IOMMUFD_DRIVER if IOMMUFD 397 398 help 398 399 Support for implementations of the ARM System MMU architecture 399 400 version 3 providing translation support to a PCIe root complex.

+1 -1

drivers/iommu/amd/io_pgtable_v2.c

reviewed

··· 158 158 159 159 __npte = set_pgtable_attr(page); 160 160 /* pte could have been changed somewhere. */ 161 161 - if (cmpxchg64(pte, __pte, __npte) != __pte) 161 161 + if (!try_cmpxchg64(pte, &__pte, __npte)) 162 162 iommu_free_page(page); 163 163 else if (IOMMU_PTE_PRESENT(__pte)) 164 164 *updated = true;

+2 -3

drivers/iommu/arm/arm-smmu-v3/Makefile

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o 3 3 - arm_smmu_v3-objs-y += arm-smmu-v3.o 4 4 - arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o 5 5 - arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) 3 3 + arm_smmu_v3-y := arm-smmu-v3.o 4 4 + arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o 6 5 7 6 obj-$(CONFIG_ARM_SMMU_V3_KUNIT_TEST) += arm-smmu-v3-test.o

+91 -340

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

reviewed

··· 13 13 #include "arm-smmu-v3.h" 14 14 #include "../../io-pgtable-arm.h" 15 15 16 16 - struct arm_smmu_mmu_notifier { 17 17 - struct mmu_notifier mn; 18 18 - struct arm_smmu_ctx_desc *cd; 19 19 - bool cleared; 20 20 - refcount_t refs; 21 21 - struct list_head list; 22 22 - struct arm_smmu_domain *domain; 23 23 - }; 24 24 - 25 25 - #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn) 26 26 - 27 27 - struct arm_smmu_bond { 28 28 - struct mm_struct *mm; 29 29 - struct arm_smmu_mmu_notifier *smmu_mn; 30 30 - struct list_head list; 31 31 - }; 32 32 - 33 33 - #define sva_to_bond(handle) \ 34 34 - container_of(handle, struct arm_smmu_bond, sva) 35 35 - 36 16 static DEFINE_MUTEX(sva_lock); 37 17 38 38 - static void 18 18 + static void __maybe_unused 39 19 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) 40 20 { 41 41 - struct arm_smmu_master *master; 21 21 + struct arm_smmu_master_domain *master_domain; 42 22 struct arm_smmu_cd target_cd; 43 23 unsigned long flags; 44 24 45 25 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 46 46 - list_for_each_entry(master, &smmu_domain->devices, domain_head) { 26 26 + list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { 27 27 + struct arm_smmu_master *master = master_domain->master; 47 28 struct arm_smmu_cd *cdptr; 48 29 49 49 - /* S1 domains only support RID attachment right now */ 50 50 - cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); 30 30 + cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 51 31 if (WARN_ON(!cdptr)) 52 32 continue; 53 33 54 34 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 55 55 - arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, 35 35 + arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 56 36 &target_cd); 57 37 } 58 38 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 59 59 - } 60 60 - 61 61 - /* 62 62 - * Check if the CPU ASID is available on the SMMU side. If a private context 63 63 - * descriptor is using it, try to replace it. 64 64 - */ 65 65 - static struct arm_smmu_ctx_desc * 66 66 - arm_smmu_share_asid(struct mm_struct *mm, u16 asid) 67 67 - { 68 68 - int ret; 69 69 - u32 new_asid; 70 70 - struct arm_smmu_ctx_desc *cd; 71 71 - struct arm_smmu_device *smmu; 72 72 - struct arm_smmu_domain *smmu_domain; 73 73 - 74 74 - cd = xa_load(&arm_smmu_asid_xa, asid); 75 75 - if (!cd) 76 76 - return NULL; 77 77 - 78 78 - if (cd->mm) { 79 79 - if (WARN_ON(cd->mm != mm)) 80 80 - return ERR_PTR(-EINVAL); 81 81 - /* All devices bound to this mm use the same cd struct. */ 82 82 - refcount_inc(&cd->refs); 83 83 - return cd; 84 84 - } 85 85 - 86 86 - smmu_domain = container_of(cd, struct arm_smmu_domain, cd); 87 87 - smmu = smmu_domain->smmu; 88 88 - 89 89 - ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd, 90 90 - XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 91 91 - if (ret) 92 92 - return ERR_PTR(-ENOSPC); 93 93 - /* 94 94 - * Race with unmap: TLB invalidations will start targeting the new ASID, 95 95 - * which isn't assigned yet. We'll do an invalidate-all on the old ASID 96 96 - * later, so it doesn't matter. 97 97 - */ 98 98 - cd->asid = new_asid; 99 99 - /* 100 100 - * Update ASID and invalidate CD in all associated masters. There will 101 101 - * be some overlap between use of both ASIDs, until we invalidate the 102 102 - * TLB. 103 103 - */ 104 104 - arm_smmu_update_s1_domain_cd_entry(smmu_domain); 105 105 - 106 106 - /* Invalidate TLB entries previously associated with that context */ 107 107 - arm_smmu_tlb_inv_asid(smmu, asid); 108 108 - 109 109 - xa_erase(&arm_smmu_asid_xa, asid); 110 110 - return NULL; 111 39 } 112 40 113 41 static u64 page_size_to_cd(void) ··· 115 187 } 116 188 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); 117 189 118 118 - static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) 119 119 - { 120 120 - u16 asid; 121 121 - int err = 0; 122 122 - struct arm_smmu_ctx_desc *cd; 123 123 - struct arm_smmu_ctx_desc *ret = NULL; 124 124 - 125 125 - /* Don't free the mm until we release the ASID */ 126 126 - mmgrab(mm); 127 127 - 128 128 - asid = arm64_mm_context_get(mm); 129 129 - if (!asid) { 130 130 - err = -ESRCH; 131 131 - goto out_drop_mm; 132 132 - } 133 133 - 134 134 - cd = kzalloc(sizeof(*cd), GFP_KERNEL); 135 135 - if (!cd) { 136 136 - err = -ENOMEM; 137 137 - goto out_put_context; 138 138 - } 139 139 - 140 140 - refcount_set(&cd->refs, 1); 141 141 - 142 142 - mutex_lock(&arm_smmu_asid_lock); 143 143 - ret = arm_smmu_share_asid(mm, asid); 144 144 - if (ret) { 145 145 - mutex_unlock(&arm_smmu_asid_lock); 146 146 - goto out_free_cd; 147 147 - } 148 148 - 149 149 - err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL); 150 150 - mutex_unlock(&arm_smmu_asid_lock); 151 151 - 152 152 - if (err) 153 153 - goto out_free_asid; 154 154 - 155 155 - cd->asid = asid; 156 156 - cd->mm = mm; 157 157 - 158 158 - return cd; 159 159 - 160 160 - out_free_asid: 161 161 - arm_smmu_free_asid(cd); 162 162 - out_free_cd: 163 163 - kfree(cd); 164 164 - out_put_context: 165 165 - arm64_mm_context_put(mm); 166 166 - out_drop_mm: 167 167 - mmdrop(mm); 168 168 - return err < 0 ? ERR_PTR(err) : ret; 169 169 - } 170 170 - 171 171 - static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 172 172 - { 173 173 - if (arm_smmu_free_asid(cd)) { 174 174 - /* Unpin ASID */ 175 175 - arm64_mm_context_put(cd->mm); 176 176 - mmdrop(cd->mm); 177 177 - kfree(cd); 178 178 - } 179 179 - } 180 180 - 181 190 /* 182 191 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this 183 192 * is used as a threshold to replace per-page TLBI commands to issue in the ··· 129 264 unsigned long start, 130 265 unsigned long end) 131 266 { 132 132 - struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 133 133 - struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 267 267 + struct arm_smmu_domain *smmu_domain = 268 268 + container_of(mn, struct arm_smmu_domain, mmu_notifier); 134 269 size_t size; 135 270 136 271 /* ··· 147 282 size = 0; 148 283 } 149 284 150 150 - if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) { 151 151 - if (!size) 152 152 - arm_smmu_tlb_inv_asid(smmu_domain->smmu, 153 153 - smmu_mn->cd->asid); 154 154 - else 155 155 - arm_smmu_tlb_inv_range_asid(start, size, 156 156 - smmu_mn->cd->asid, 157 157 - PAGE_SIZE, false, 158 158 - smmu_domain); 159 159 - } 285 285 + if (!size) 286 286 + arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 287 287 + else 288 288 + arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, 289 289 + PAGE_SIZE, false, smmu_domain); 160 290 161 161 - arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), start, 162 162 - size); 291 291 + arm_smmu_atc_inv_domain(smmu_domain, start, size); 163 292 } 164 293 165 294 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 166 295 { 167 167 - struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); 168 168 - struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 169 169 - struct arm_smmu_master *master; 296 296 + struct arm_smmu_domain *smmu_domain = 297 297 + container_of(mn, struct arm_smmu_domain, mmu_notifier); 298 298 + struct arm_smmu_master_domain *master_domain; 170 299 unsigned long flags; 171 171 - 172 172 - mutex_lock(&sva_lock); 173 173 - if (smmu_mn->cleared) { 174 174 - mutex_unlock(&sva_lock); 175 175 - return; 176 176 - } 177 300 178 301 /* 179 302 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 180 303 * but disable translation. 181 304 */ 182 305 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 183 183 - list_for_each_entry(master, &smmu_domain->devices, domain_head) { 306 306 + list_for_each_entry(master_domain, &smmu_domain->devices, 307 307 + devices_elm) { 308 308 + struct arm_smmu_master *master = master_domain->master; 184 309 struct arm_smmu_cd target; 185 310 struct arm_smmu_cd *cdptr; 186 311 187 187 - cdptr = arm_smmu_get_cd_ptr(master, mm_get_enqcmd_pasid(mm)); 312 312 + cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 188 313 if (WARN_ON(!cdptr)) 189 314 continue; 190 190 - arm_smmu_make_sva_cd(&target, master, NULL, smmu_mn->cd->asid); 191 191 - arm_smmu_write_cd_entry(master, mm_get_enqcmd_pasid(mm), cdptr, 315 315 + arm_smmu_make_sva_cd(&target, master, NULL, 316 316 + smmu_domain->cd.asid); 317 317 + arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 192 318 &target); 193 319 } 194 320 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 195 321 196 196 - arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); 197 197 - arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 0); 198 198 - 199 199 - smmu_mn->cleared = true; 200 200 - mutex_unlock(&sva_lock); 322 322 + arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 323 323 + arm_smmu_atc_inv_domain(smmu_domain, 0, 0); 201 324 } 202 325 203 326 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 204 327 { 205 205 - kfree(mn_to_smmu(mn)); 328 328 + kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier)); 206 329 } 207 330 208 331 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { ··· 198 345 .release = arm_smmu_mm_release, 199 346 .free_notifier = arm_smmu_mmu_notifier_free, 200 347 }; 201 201 - 202 202 - /* Allocate or get existing MMU notifier for this {domain, mm} pair */ 203 203 - static struct arm_smmu_mmu_notifier * 204 204 - arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain, 205 205 - struct mm_struct *mm) 206 206 - { 207 207 - int ret; 208 208 - struct arm_smmu_ctx_desc *cd; 209 209 - struct arm_smmu_mmu_notifier *smmu_mn; 210 210 - 211 211 - list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) { 212 212 - if (smmu_mn->mn.mm == mm) { 213 213 - refcount_inc(&smmu_mn->refs); 214 214 - return smmu_mn; 215 215 - } 216 216 - } 217 217 - 218 218 - cd = arm_smmu_alloc_shared_cd(mm); 219 219 - if (IS_ERR(cd)) 220 220 - return ERR_CAST(cd); 221 221 - 222 222 - smmu_mn = kzalloc(sizeof(*smmu_mn), GFP_KERNEL); 223 223 - if (!smmu_mn) { 224 224 - ret = -ENOMEM; 225 225 - goto err_free_cd; 226 226 - } 227 227 - 228 228 - refcount_set(&smmu_mn->refs, 1); 229 229 - smmu_mn->cd = cd; 230 230 - smmu_mn->domain = smmu_domain; 231 231 - smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops; 232 232 - 233 233 - ret = mmu_notifier_register(&smmu_mn->mn, mm); 234 234 - if (ret) { 235 235 - kfree(smmu_mn); 236 236 - goto err_free_cd; 237 237 - } 238 238 - 239 239 - list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers); 240 240 - return smmu_mn; 241 241 - 242 242 - err_free_cd: 243 243 - arm_smmu_free_shared_cd(cd); 244 244 - return ERR_PTR(ret); 245 245 - } 246 246 - 247 247 - static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) 248 248 - { 249 249 - struct mm_struct *mm = smmu_mn->mn.mm; 250 250 - struct arm_smmu_ctx_desc *cd = smmu_mn->cd; 251 251 - struct arm_smmu_domain *smmu_domain = smmu_mn->domain; 252 252 - 253 253 - if (!refcount_dec_and_test(&smmu_mn->refs)) 254 254 - return; 255 255 - 256 256 - list_del(&smmu_mn->list); 257 257 - 258 258 - /* 259 259 - * If we went through clear(), we've already invalidated, and no 260 260 - * new TLB entry can have been formed. 261 261 - */ 262 262 - if (!smmu_mn->cleared) { 263 263 - arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid); 264 264 - arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 265 265 - 0); 266 266 - } 267 267 - 268 268 - /* Frees smmu_mn */ 269 269 - mmu_notifier_put(&smmu_mn->mn); 270 270 - arm_smmu_free_shared_cd(cd); 271 271 - } 272 272 - 273 273 - static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, 274 274 - struct mm_struct *mm) 275 275 - { 276 276 - int ret; 277 277 - struct arm_smmu_cd target; 278 278 - struct arm_smmu_cd *cdptr; 279 279 - struct arm_smmu_bond *bond; 280 280 - struct arm_smmu_master *master = dev_iommu_priv_get(dev); 281 281 - struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 282 282 - struct arm_smmu_domain *smmu_domain; 283 283 - 284 284 - if (!(domain->type & __IOMMU_DOMAIN_PAGING)) 285 285 - return -ENODEV; 286 286 - smmu_domain = to_smmu_domain(domain); 287 287 - if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) 288 288 - return -ENODEV; 289 289 - 290 290 - if (!master || !master->sva_enabled) 291 291 - return -ENODEV; 292 292 - 293 293 - bond = kzalloc(sizeof(*bond), GFP_KERNEL); 294 294 - if (!bond) 295 295 - return -ENOMEM; 296 296 - 297 297 - bond->mm = mm; 298 298 - 299 299 - bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm); 300 300 - if (IS_ERR(bond->smmu_mn)) { 301 301 - ret = PTR_ERR(bond->smmu_mn); 302 302 - goto err_free_bond; 303 303 - } 304 304 - 305 305 - cdptr = arm_smmu_alloc_cd_ptr(master, mm_get_enqcmd_pasid(mm)); 306 306 - if (!cdptr) { 307 307 - ret = -ENOMEM; 308 308 - goto err_put_notifier; 309 309 - } 310 310 - arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid); 311 311 - arm_smmu_write_cd_entry(master, pasid, cdptr, &target); 312 312 - 313 313 - list_add(&bond->list, &master->bonds); 314 314 - return 0; 315 315 - 316 316 - err_put_notifier: 317 317 - arm_smmu_mmu_notifier_put(bond->smmu_mn); 318 318 - err_free_bond: 319 319 - kfree(bond); 320 320 - return ret; 321 321 - } 322 348 323 349 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 324 350 { ··· 315 583 int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 316 584 { 317 585 mutex_lock(&sva_lock); 318 318 - if (!list_empty(&master->bonds)) { 319 319 - dev_err(master->dev, "cannot disable SVA, device is bound\n"); 320 320 - mutex_unlock(&sva_lock); 321 321 - return -EBUSY; 322 322 - } 323 586 arm_smmu_master_sva_disable_iopf(master); 324 587 master->sva_enabled = false; 325 588 mutex_unlock(&sva_lock); ··· 331 604 mmu_notifier_synchronize(); 332 605 } 333 606 334 334 - void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 335 335 - struct device *dev, ioasid_t id) 336 336 - { 337 337 - struct mm_struct *mm = domain->mm; 338 338 - struct arm_smmu_bond *bond = NULL, *t; 339 339 - struct arm_smmu_master *master = dev_iommu_priv_get(dev); 340 340 - 341 341 - mutex_lock(&sva_lock); 342 342 - 343 343 - arm_smmu_clear_cd(master, id); 344 344 - 345 345 - list_for_each_entry(t, &master->bonds, list) { 346 346 - if (t->mm == mm) { 347 347 - bond = t; 348 348 - break; 349 349 - } 350 350 - } 351 351 - 352 352 - if (!WARN_ON(!bond)) { 353 353 - list_del(&bond->list); 354 354 - arm_smmu_mmu_notifier_put(bond->smmu_mn); 355 355 - kfree(bond); 356 356 - } 357 357 - mutex_unlock(&sva_lock); 358 358 - } 359 359 - 360 607 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 361 608 struct device *dev, ioasid_t id) 362 609 { 363 363 - int ret = 0; 364 364 - struct mm_struct *mm = domain->mm; 610 610 + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 611 611 + struct arm_smmu_master *master = dev_iommu_priv_get(dev); 612 612 + struct arm_smmu_cd target; 613 613 + int ret; 365 614 366 366 - if (mm_get_enqcmd_pasid(mm) != id) 615 615 + /* Prevent arm_smmu_mm_release from being called while we are attaching */ 616 616 + if (!mmget_not_zero(domain->mm)) 367 617 return -EINVAL; 368 618 369 369 - mutex_lock(&sva_lock); 370 370 - ret = __arm_smmu_sva_bind(dev, id, mm); 371 371 - mutex_unlock(&sva_lock); 619 619 + /* 620 620 + * This does not need the arm_smmu_asid_lock because SVA domains never 621 621 + * get reassigned 622 622 + */ 623 623 + arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid); 624 624 + ret = arm_smmu_set_pasid(master, smmu_domain, id, &target); 372 625 626 626 + mmput(domain->mm); 373 627 return ret; 374 628 } 375 629 376 630 static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 377 631 { 378 378 - kfree(domain); 632 632 + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 633 633 + 634 634 + /* 635 635 + * Ensure the ASID is empty in the iommu cache before allowing reuse. 636 636 + */ 637 637 + arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 638 638 + 639 639 + /* 640 640 + * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can 641 641 + * still be called/running at this point. We allow the ASID to be 642 642 + * reused, and if there is a race then it just suffers harmless 643 643 + * unnecessary invalidation. 644 644 + */ 645 645 + xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 646 646 + 647 647 + /* 648 648 + * Actual free is defered to the SRCU callback 649 649 + * arm_smmu_mmu_notifier_free() 650 650 + */ 651 651 + mmu_notifier_put(&smmu_domain->mmu_notifier); 379 652 } 380 653 381 654 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { ··· 383 656 .free = arm_smmu_sva_domain_free 384 657 }; 385 658 386 386 - struct iommu_domain *arm_smmu_sva_domain_alloc(void) 659 659 + struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, 660 660 + struct mm_struct *mm) 387 661 { 388 388 - struct iommu_domain *domain; 662 662 + struct arm_smmu_master *master = dev_iommu_priv_get(dev); 663 663 + struct arm_smmu_device *smmu = master->smmu; 664 664 + struct arm_smmu_domain *smmu_domain; 665 665 + u32 asid; 666 666 + int ret; 389 667 390 390 - domain = kzalloc(sizeof(*domain), GFP_KERNEL); 391 391 - if (!domain) 392 392 - return NULL; 393 393 - domain->ops = &arm_smmu_sva_domain_ops; 668 668 + smmu_domain = arm_smmu_domain_alloc(); 669 669 + if (IS_ERR(smmu_domain)) 670 670 + return ERR_CAST(smmu_domain); 671 671 + smmu_domain->domain.type = IOMMU_DOMAIN_SVA; 672 672 + smmu_domain->domain.ops = &arm_smmu_sva_domain_ops; 673 673 + smmu_domain->smmu = smmu; 394 674 395 395 - return domain; 675 675 + ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, 676 676 + XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 677 677 + if (ret) 678 678 + goto err_free; 679 679 + 680 680 + smmu_domain->cd.asid = asid; 681 681 + smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops; 682 682 + ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm); 683 683 + if (ret) 684 684 + goto err_asid; 685 685 + 686 686 + return &smmu_domain->domain; 687 687 + 688 688 + err_asid: 689 689 + xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 690 690 + err_free: 691 691 + kfree(smmu_domain); 692 692 + return ERR_PTR(ret); 396 693 }

+110 -7

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c

reviewed

··· 144 144 KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); 145 145 } 146 146 147 147 + static void arm_smmu_v3_test_ste_expect_non_hitless_transition( 148 148 + struct kunit *test, const struct arm_smmu_ste *cur, 149 149 + const struct arm_smmu_ste *target, unsigned int num_syncs_expected) 150 150 + { 151 151 + arm_smmu_v3_test_ste_expect_transition(test, cur, target, 152 152 + num_syncs_expected, false); 153 153 + } 154 154 + 147 155 static void arm_smmu_v3_test_ste_expect_hitless_transition( 148 156 struct kunit *test, const struct arm_smmu_ste *cur, 149 157 const struct arm_smmu_ste *target, unsigned int num_syncs_expected) ··· 163 155 static const dma_addr_t fake_cdtab_dma_addr = 0xF0F0F0F0F0F0; 164 156 165 157 static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, 158 158 + unsigned int s1dss, 166 159 const dma_addr_t dma_addr) 167 160 { 168 161 struct arm_smmu_master master = { ··· 173 164 .smmu = &smmu, 174 165 }; 175 166 176 176 - arm_smmu_make_cdtable_ste(ste, &master); 167 167 + arm_smmu_make_cdtable_ste(ste, &master, true, s1dss); 177 168 } 178 169 179 170 static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test) ··· 203 194 { 204 195 struct arm_smmu_ste ste; 205 196 206 206 - arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 197 197 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 198 198 + fake_cdtab_dma_addr); 207 199 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, 208 200 NUM_EXPECTED_SYNCS(2)); 209 201 } ··· 213 203 { 214 204 struct arm_smmu_ste ste; 215 205 216 216 - arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 206 206 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 207 207 + fake_cdtab_dma_addr); 217 208 arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, 218 209 NUM_EXPECTED_SYNCS(2)); 219 210 } ··· 223 212 { 224 213 struct arm_smmu_ste ste; 225 214 226 226 - arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 215 215 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 216 216 + fake_cdtab_dma_addr); 227 217 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, 228 218 NUM_EXPECTED_SYNCS(3)); 229 219 } ··· 233 221 { 234 222 struct arm_smmu_ste ste; 235 223 236 236 - arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 224 224 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 225 225 + fake_cdtab_dma_addr); 237 226 arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, 238 227 NUM_EXPECTED_SYNCS(3)); 228 228 + } 229 229 + 230 230 + static void arm_smmu_v3_write_ste_test_cdtable_s1dss_change(struct kunit *test) 231 231 + { 232 232 + struct arm_smmu_ste ste; 233 233 + struct arm_smmu_ste s1dss_bypass; 234 234 + 235 235 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 236 236 + fake_cdtab_dma_addr); 237 237 + arm_smmu_test_make_cdtable_ste(&s1dss_bypass, STRTAB_STE_1_S1DSS_BYPASS, 238 238 + fake_cdtab_dma_addr); 239 239 + 240 240 + /* 241 241 + * Flipping s1dss on a CD table STE only involves changes to the second 242 242 + * qword of an STE and can be done in a single write. 243 243 + */ 244 244 + arm_smmu_v3_test_ste_expect_hitless_transition( 245 245 + test, &ste, &s1dss_bypass, NUM_EXPECTED_SYNCS(1)); 246 246 + arm_smmu_v3_test_ste_expect_hitless_transition( 247 247 + test, &s1dss_bypass, &ste, NUM_EXPECTED_SYNCS(1)); 248 248 + } 249 249 + 250 250 + static void 251 251 + arm_smmu_v3_write_ste_test_s1dssbypass_to_stebypass(struct kunit *test) 252 252 + { 253 253 + struct arm_smmu_ste s1dss_bypass; 254 254 + 255 255 + arm_smmu_test_make_cdtable_ste(&s1dss_bypass, STRTAB_STE_1_S1DSS_BYPASS, 256 256 + fake_cdtab_dma_addr); 257 257 + arm_smmu_v3_test_ste_expect_hitless_transition( 258 258 + test, &s1dss_bypass, &bypass_ste, NUM_EXPECTED_SYNCS(2)); 259 259 + } 260 260 + 261 261 + static void 262 262 + arm_smmu_v3_write_ste_test_stebypass_to_s1dssbypass(struct kunit *test) 263 263 + { 264 264 + struct arm_smmu_ste s1dss_bypass; 265 265 + 266 266 + arm_smmu_test_make_cdtable_ste(&s1dss_bypass, STRTAB_STE_1_S1DSS_BYPASS, 267 267 + fake_cdtab_dma_addr); 268 268 + arm_smmu_v3_test_ste_expect_hitless_transition( 269 269 + test, &bypass_ste, &s1dss_bypass, NUM_EXPECTED_SYNCS(2)); 239 270 } 240 271 241 272 static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste, ··· 286 231 { 287 232 struct arm_smmu_master master = { 288 233 .smmu = &smmu, 289 289 - .ats_enabled = ats_enabled, 290 234 }; 291 235 struct io_pgtable io_pgtable = {}; 292 236 struct arm_smmu_domain smmu_domain = { ··· 301 247 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sl = 3; 302 248 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tsz = 4; 303 249 304 304 - arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain); 250 250 + arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain, ats_enabled); 305 251 } 306 252 307 253 static void arm_smmu_v3_write_ste_test_s2_to_abort(struct kunit *test) ··· 338 284 arm_smmu_test_make_s2_ste(&ste, true); 339 285 arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, 340 286 NUM_EXPECTED_SYNCS(2)); 287 287 + } 288 288 + 289 289 + static void arm_smmu_v3_write_ste_test_s1_to_s2(struct kunit *test) 290 290 + { 291 291 + struct arm_smmu_ste s1_ste; 292 292 + struct arm_smmu_ste s2_ste; 293 293 + 294 294 + arm_smmu_test_make_cdtable_ste(&s1_ste, STRTAB_STE_1_S1DSS_SSID0, 295 295 + fake_cdtab_dma_addr); 296 296 + arm_smmu_test_make_s2_ste(&s2_ste, true); 297 297 + arm_smmu_v3_test_ste_expect_hitless_transition(test, &s1_ste, &s2_ste, 298 298 + NUM_EXPECTED_SYNCS(3)); 299 299 + } 300 300 + 301 301 + static void arm_smmu_v3_write_ste_test_s2_to_s1(struct kunit *test) 302 302 + { 303 303 + struct arm_smmu_ste s1_ste; 304 304 + struct arm_smmu_ste s2_ste; 305 305 + 306 306 + arm_smmu_test_make_cdtable_ste(&s1_ste, STRTAB_STE_1_S1DSS_SSID0, 307 307 + fake_cdtab_dma_addr); 308 308 + arm_smmu_test_make_s2_ste(&s2_ste, true); 309 309 + arm_smmu_v3_test_ste_expect_hitless_transition(test, &s2_ste, &s1_ste, 310 310 + NUM_EXPECTED_SYNCS(3)); 311 311 + } 312 312 + 313 313 + static void arm_smmu_v3_write_ste_test_non_hitless(struct kunit *test) 314 314 + { 315 315 + struct arm_smmu_ste ste; 316 316 + struct arm_smmu_ste ste_2; 317 317 + 318 318 + /* 319 319 + * Although no flow resembles this in practice, one way to force an STE 320 320 + * update to be non-hitless is to change its CD table pointer as well as 321 321 + * s1 dss field in the same update. 322 322 + */ 323 323 + arm_smmu_test_make_cdtable_ste(&ste, STRTAB_STE_1_S1DSS_SSID0, 324 324 + fake_cdtab_dma_addr); 325 325 + arm_smmu_test_make_cdtable_ste(&ste_2, STRTAB_STE_1_S1DSS_BYPASS, 326 326 + 0x4B4B4b4B4B); 327 327 + arm_smmu_v3_test_ste_expect_non_hitless_transition( 328 328 + test, &ste, &ste_2, NUM_EXPECTED_SYNCS(3)); 341 329 } 342 330 343 331 static void arm_smmu_v3_test_cd_expect_transition( ··· 535 439 KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_cdtable), 536 440 KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_bypass), 537 441 KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_cdtable), 442 442 + KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_s1dss_change), 443 443 + KUNIT_CASE(arm_smmu_v3_write_ste_test_s1dssbypass_to_stebypass), 444 444 + KUNIT_CASE(arm_smmu_v3_write_ste_test_stebypass_to_s1dssbypass), 538 445 KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_abort), 539 446 KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_s2), 540 447 KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_bypass), 541 448 KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_s2), 449 449 + KUNIT_CASE(arm_smmu_v3_write_ste_test_s1_to_s2), 450 450 + KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_s1), 451 451 + KUNIT_CASE(arm_smmu_v3_write_ste_test_non_hitless), 542 452 KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_clear), 543 453 KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid), 544 454 KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), ··· 567 465 kunit_test_suites(&arm_smmu_v3_test_module); 568 466 569 467 MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING); 468 468 + MODULE_DESCRIPTION("KUnit tests for arm-smmu-v3 driver"); 570 469 MODULE_LICENSE("GPL v2");

+580 -179

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

reviewed

··· 27 27 #include <linux/pci-ats.h> 28 28 #include <linux/platform_device.h> 29 29 #include <kunit/visibility.h> 30 30 + #include <uapi/linux/iommufd.h> 30 31 31 32 #include "arm-smmu-v3.h" 32 33 #include "../../dma-iommu.h" ··· 36 35 module_param(disable_msipolling, bool, 0444); 37 36 MODULE_PARM_DESC(disable_msipolling, 38 37 "Disable MSI-based polling for CMD_SYNC completion."); 38 38 + 39 39 + static struct iommu_ops arm_smmu_ops; 40 40 + static struct iommu_dirty_ops arm_smmu_dirty_ops; 39 41 40 42 enum arm_smmu_msi_index { 41 43 EVTQ_MSI_INDEX, ··· 84 80 }; 85 81 86 82 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, 87 87 - struct arm_smmu_device *smmu); 83 83 + struct arm_smmu_device *smmu, u32 flags); 88 84 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master); 89 85 90 86 static void parse_driver_options(struct arm_smmu_device *smmu) ··· 995 991 STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | 996 992 STRTAB_STE_1_EATS); 997 993 used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); 994 994 + 995 995 + /* 996 996 + * See 13.5 Summary of attribute/permission configuration fields 997 997 + * for the SHCFG behavior. 998 998 + */ 999 999 + if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) == 1000 1000 + STRTAB_STE_1_S1DSS_BYPASS) 1001 1001 + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); 998 1002 } 999 1003 1000 1004 /* S2 translates */ ··· 1223 1211 return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES]; 1224 1212 } 1225 1213 1226 1226 - struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, 1227 1227 - u32 ssid) 1214 1214 + static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, 1215 1215 + u32 ssid) 1228 1216 { 1229 1217 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; 1230 1218 struct arm_smmu_device *smmu = master->smmu; ··· 1301 1289 struct arm_smmu_cd *cdptr, 1302 1290 const struct arm_smmu_cd *target) 1303 1291 { 1292 1292 + bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V); 1293 1293 + bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V); 1304 1294 struct arm_smmu_cd_writer cd_writer = { 1305 1295 .writer = { 1306 1296 .ops = &arm_smmu_cd_writer_ops, ··· 1310 1296 }, 1311 1297 .ssid = ssid, 1312 1298 }; 1299 1299 + 1300 1300 + if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) { 1301 1301 + if (cur_valid) 1302 1302 + master->cd_table.used_ssids--; 1303 1303 + else 1304 1304 + master->cd_table.used_ssids++; 1305 1305 + } 1313 1306 1314 1307 arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data); 1315 1308 } ··· 1352 1331 CTXDESC_CD_0_ASET | 1353 1332 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) 1354 1333 ); 1334 1334 + 1335 1335 + /* To enable dirty flag update, set both Access flag and dirty state update */ 1336 1336 + if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD) 1337 1337 + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA | 1338 1338 + CTXDESC_CD_0_TCR_HD); 1339 1339 + 1355 1340 target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr & 1356 1341 CTXDESC_CD_1_TTB0_MASK); 1357 1342 target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); ··· 1457 1430 cd_table->cdtab = NULL; 1458 1431 } 1459 1432 1460 1460 - bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd) 1461 1461 - { 1462 1462 - bool free; 1463 1463 - struct arm_smmu_ctx_desc *old_cd; 1464 1464 - 1465 1465 - if (!cd->asid) 1466 1466 - return false; 1467 1467 - 1468 1468 - free = refcount_dec_and_test(&cd->refs); 1469 1469 - if (free) { 1470 1470 - old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid); 1471 1471 - WARN_ON(old_cd != cd); 1472 1472 - } 1473 1473 - return free; 1474 1474 - } 1475 1475 - 1476 1433 /* Stream table manipulation functions */ 1477 1477 - static void 1478 1478 - arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) 1434 1434 + static void arm_smmu_write_strtab_l1_desc(__le64 *dst, dma_addr_t l2ptr_dma) 1479 1435 { 1480 1436 u64 val = 0; 1481 1437 1482 1482 - val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); 1483 1483 - val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; 1438 1438 + val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1); 1439 1439 + val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; 1484 1440 1485 1441 /* The HW has 64 bit atomicity with stores to the L2 STE table */ 1486 1442 WRITE_ONCE(*dst, cpu_to_le64(val)); ··· 1548 1538 1549 1539 VISIBLE_IF_KUNIT 1550 1540 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, 1551 1551 - struct arm_smmu_master *master) 1541 1541 + struct arm_smmu_master *master, bool ats_enabled, 1542 1542 + unsigned int s1dss) 1552 1543 { 1553 1544 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; 1554 1545 struct arm_smmu_device *smmu = master->smmu; ··· 1563 1552 FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax)); 1564 1553 1565 1554 target->data[1] = cpu_to_le64( 1566 1566 - FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | 1555 1555 + FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) | 1567 1556 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1568 1557 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | 1569 1558 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | ··· 1572 1561 STRTAB_STE_1_S1STALLD : 1573 1562 0) | 1574 1563 FIELD_PREP(STRTAB_STE_1_EATS, 1575 1575 - master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); 1564 1564 + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); 1565 1565 + 1566 1566 + if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) && 1567 1567 + s1dss == STRTAB_STE_1_S1DSS_BYPASS) 1568 1568 + target->data[1] |= cpu_to_le64(FIELD_PREP( 1569 1569 + STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); 1576 1570 1577 1571 if (smmu->features & ARM_SMMU_FEAT_E2H) { 1578 1572 /* ··· 1607 1591 VISIBLE_IF_KUNIT 1608 1592 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, 1609 1593 struct arm_smmu_master *master, 1610 1610 - struct arm_smmu_domain *smmu_domain) 1594 1594 + struct arm_smmu_domain *smmu_domain, 1595 1595 + bool ats_enabled) 1611 1596 { 1612 1597 struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; 1613 1598 const struct io_pgtable_cfg *pgtbl_cfg = ··· 1625 1608 1626 1609 target->data[1] = cpu_to_le64( 1627 1610 FIELD_PREP(STRTAB_STE_1_EATS, 1628 1628 - master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); 1611 1611 + ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); 1629 1612 1630 1613 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) 1631 1614 target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, ··· 1672 1655 { 1673 1656 size_t size; 1674 1657 void *strtab; 1658 1658 + dma_addr_t l2ptr_dma; 1675 1659 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 1676 1660 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; 1677 1661 ··· 1682 1664 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 1683 1665 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS]; 1684 1666 1685 1685 - desc->span = STRTAB_SPLIT + 1; 1686 1686 - desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, 1667 1667 + desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &l2ptr_dma, 1687 1668 GFP_KERNEL); 1688 1669 if (!desc->l2ptr) { 1689 1670 dev_err(smmu->dev, ··· 1692 1675 } 1693 1676 1694 1677 arm_smmu_init_initial_stes(desc->l2ptr, 1 << STRTAB_SPLIT); 1695 1695 - arm_smmu_write_strtab_l1_desc(strtab, desc); 1678 1678 + arm_smmu_write_strtab_l1_desc(strtab, l2ptr_dma); 1696 1679 return 0; 1697 1680 } 1698 1681 ··· 2012 1995 cmd->atc.size = log2_span; 2013 1996 } 2014 1997 2015 2015 - static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) 1998 1998 + static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, 1999 1999 + ioasid_t ssid) 2016 2000 { 2017 2001 int i; 2018 2002 struct arm_smmu_cmdq_ent cmd; 2019 2003 struct arm_smmu_cmdq_batch cmds; 2020 2004 2021 2021 - arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd); 2005 2005 + arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd); 2022 2006 2023 2007 cmds.num = 0; 2024 2008 for (i = 0; i < master->num_streams; i++) { ··· 2030 2012 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); 2031 2013 } 2032 2014 2033 2033 - int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 2015 2015 + int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, 2034 2016 unsigned long iova, size_t size) 2035 2017 { 2018 2018 + struct arm_smmu_master_domain *master_domain; 2036 2019 int i; 2037 2020 unsigned long flags; 2038 2021 struct arm_smmu_cmdq_ent cmd; 2039 2039 - struct arm_smmu_master *master; 2040 2022 struct arm_smmu_cmdq_batch cmds; 2041 2023 2042 2024 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) ··· 2059 2041 if (!atomic_read(&smmu_domain->nr_ats_masters)) 2060 2042 return 0; 2061 2043 2062 2062 - arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd); 2063 2063 - 2064 2044 cmds.num = 0; 2065 2045 2066 2046 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2067 2067 - list_for_each_entry(master, &smmu_domain->devices, domain_head) { 2047 2047 + list_for_each_entry(master_domain, &smmu_domain->devices, 2048 2048 + devices_elm) { 2049 2049 + struct arm_smmu_master *master = master_domain->master; 2050 2050 + 2068 2051 if (!master->ats_enabled) 2069 2052 continue; 2053 2053 + 2054 2054 + arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size, &cmd); 2070 2055 2071 2056 for (i = 0; i < master->num_streams; i++) { 2072 2057 cmd.atc.sid = master->streams[i].id; ··· 2102 2081 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; 2103 2082 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); 2104 2083 } 2105 2105 - arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0); 2084 2084 + arm_smmu_atc_inv_domain(smmu_domain, 0, 0); 2106 2085 } 2107 2086 2108 2087 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, ··· 2200 2179 * Unfortunately, this can't be leaf-only since we may have 2201 2180 * zapped an entire table. 2202 2181 */ 2203 2203 - arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, iova, size); 2182 2182 + arm_smmu_atc_inv_domain(smmu_domain, iova, size); 2204 2183 } 2205 2184 2206 2185 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, ··· 2241 2220 .tlb_add_page = arm_smmu_tlb_inv_page_nosync, 2242 2221 }; 2243 2222 2223 2223 + static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu) 2224 2224 + { 2225 2225 + u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY); 2226 2226 + 2227 2227 + return (smmu->features & features) == features; 2228 2228 + } 2229 2229 + 2244 2230 /* IOMMU API */ 2245 2231 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 2246 2232 { ··· 2260 2232 case IOMMU_CAP_NOEXEC: 2261 2233 case IOMMU_CAP_DEFERRED_FLUSH: 2262 2234 return true; 2235 2235 + case IOMMU_CAP_DIRTY_TRACKING: 2236 2236 + return arm_smmu_dbm_capable(master->smmu); 2263 2237 default: 2264 2238 return false; 2265 2239 } 2266 2240 } 2267 2241 2268 2268 - static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 2242 2242 + struct arm_smmu_domain *arm_smmu_domain_alloc(void) 2269 2243 { 2244 2244 + struct arm_smmu_domain *smmu_domain; 2270 2245 2271 2271 - if (type == IOMMU_DOMAIN_SVA) 2272 2272 - return arm_smmu_sva_domain_alloc(); 2273 2273 - return ERR_PTR(-EOPNOTSUPP); 2246 2246 + smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 2247 2247 + if (!smmu_domain) 2248 2248 + return ERR_PTR(-ENOMEM); 2249 2249 + 2250 2250 + mutex_init(&smmu_domain->init_mutex); 2251 2251 + INIT_LIST_HEAD(&smmu_domain->devices); 2252 2252 + spin_lock_init(&smmu_domain->devices_lock); 2253 2253 + 2254 2254 + return smmu_domain; 2274 2255 } 2275 2256 2276 2257 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) ··· 2291 2254 * We can't really do anything meaningful until we've added a 2292 2255 * master. 2293 2256 */ 2294 2294 - smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 2295 2295 - if (!smmu_domain) 2296 2296 - return ERR_PTR(-ENOMEM); 2297 2297 - 2298 2298 - mutex_init(&smmu_domain->init_mutex); 2299 2299 - INIT_LIST_HEAD(&smmu_domain->devices); 2300 2300 - spin_lock_init(&smmu_domain->devices_lock); 2301 2301 - INIT_LIST_HEAD(&smmu_domain->mmu_notifiers); 2257 2257 + smmu_domain = arm_smmu_domain_alloc(); 2258 2258 + if (IS_ERR(smmu_domain)) 2259 2259 + return ERR_CAST(smmu_domain); 2302 2260 2303 2261 if (dev) { 2304 2262 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2305 2263 int ret; 2306 2264 2307 2307 - ret = arm_smmu_domain_finalise(smmu_domain, master->smmu); 2265 2265 + ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, 0); 2308 2266 if (ret) { 2309 2267 kfree(smmu_domain); 2310 2268 return ERR_PTR(ret); ··· 2308 2276 return &smmu_domain->domain; 2309 2277 } 2310 2278 2311 2311 - static void arm_smmu_domain_free(struct iommu_domain *domain) 2279 2279 + static void arm_smmu_domain_free_paging(struct iommu_domain *domain) 2312 2280 { 2313 2281 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2314 2282 struct arm_smmu_device *smmu = smmu_domain->smmu; ··· 2319 2287 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 2320 2288 /* Prevent SVA from touching the CD while we're freeing it */ 2321 2289 mutex_lock(&arm_smmu_asid_lock); 2322 2322 - arm_smmu_free_asid(&smmu_domain->cd); 2290 2290 + xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 2323 2291 mutex_unlock(&arm_smmu_asid_lock); 2324 2292 } else { 2325 2293 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; ··· 2334 2302 struct arm_smmu_domain *smmu_domain) 2335 2303 { 2336 2304 int ret; 2337 2337 - u32 asid; 2305 2305 + u32 asid = 0; 2338 2306 struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; 2339 2339 - 2340 2340 - refcount_set(&cd->refs, 1); 2341 2307 2342 2308 /* Prevent SVA from modifying the ASID until it is written to the CD */ 2343 2309 mutex_lock(&arm_smmu_asid_lock); 2344 2344 - ret = xa_alloc(&arm_smmu_asid_xa, &asid, cd, 2310 2310 + ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, 2345 2311 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 2346 2312 cd->asid = (u16)asid; 2347 2313 mutex_unlock(&arm_smmu_asid_lock); ··· 2363 2333 } 2364 2334 2365 2335 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, 2366 2366 - struct arm_smmu_device *smmu) 2336 2336 + struct arm_smmu_device *smmu, u32 flags) 2367 2337 { 2368 2338 int ret; 2369 2369 - unsigned long ias, oas; 2370 2339 enum io_pgtable_fmt fmt; 2371 2340 struct io_pgtable_cfg pgtbl_cfg; 2372 2341 struct io_pgtable_ops *pgtbl_ops; 2373 2342 int (*finalise_stage_fn)(struct arm_smmu_device *smmu, 2374 2343 struct arm_smmu_domain *smmu_domain); 2344 2344 + bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 2375 2345 2376 2346 /* Restrict the stage to what we can actually support */ 2377 2347 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) ··· 2379 2349 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 2380 2350 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 2381 2351 2352 2352 + pgtbl_cfg = (struct io_pgtable_cfg) { 2353 2353 + .pgsize_bitmap = smmu->pgsize_bitmap, 2354 2354 + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, 2355 2355 + .tlb = &arm_smmu_flush_ops, 2356 2356 + .iommu_dev = smmu->dev, 2357 2357 + }; 2358 2358 + 2382 2359 switch (smmu_domain->stage) { 2383 2383 - case ARM_SMMU_DOMAIN_S1: 2384 2384 - ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; 2385 2385 - ias = min_t(unsigned long, ias, VA_BITS); 2386 2386 - oas = smmu->ias; 2360 2360 + case ARM_SMMU_DOMAIN_S1: { 2361 2361 + unsigned long ias = (smmu->features & 2362 2362 + ARM_SMMU_FEAT_VAX) ? 52 : 48; 2363 2363 + 2364 2364 + pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS); 2365 2365 + pgtbl_cfg.oas = smmu->ias; 2366 2366 + if (enable_dirty) 2367 2367 + pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD; 2387 2368 fmt = ARM_64_LPAE_S1; 2388 2369 finalise_stage_fn = arm_smmu_domain_finalise_s1; 2389 2370 break; 2371 2371 + } 2390 2372 case ARM_SMMU_DOMAIN_S2: 2391 2391 - ias = smmu->ias; 2392 2392 - oas = smmu->oas; 2373 2373 + if (enable_dirty) 2374 2374 + return -EOPNOTSUPP; 2375 2375 + pgtbl_cfg.ias = smmu->ias; 2376 2376 + pgtbl_cfg.oas = smmu->oas; 2393 2377 fmt = ARM_64_LPAE_S2; 2394 2378 finalise_stage_fn = arm_smmu_domain_finalise_s2; 2395 2379 break; 2396 2380 default: 2397 2381 return -EINVAL; 2398 2382 } 2399 2399 - 2400 2400 - pgtbl_cfg = (struct io_pgtable_cfg) { 2401 2401 - .pgsize_bitmap = smmu->pgsize_bitmap, 2402 2402 - .ias = ias, 2403 2403 - .oas = oas, 2404 2404 - .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, 2405 2405 - .tlb = &arm_smmu_flush_ops, 2406 2406 - .iommu_dev = smmu->dev, 2407 2407 - }; 2408 2383 2409 2384 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 2410 2385 if (!pgtbl_ops) ··· 2418 2383 smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 2419 2384 smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; 2420 2385 smmu_domain->domain.geometry.force_aperture = true; 2386 2386 + if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 2387 2387 + smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops; 2421 2388 2422 2389 ret = finalise_stage_fn(smmu, smmu_domain); 2423 2390 if (ret < 0) { ··· 2457 2420 int i, j; 2458 2421 struct arm_smmu_device *smmu = master->smmu; 2459 2422 2423 2423 + master->cd_table.in_ste = 2424 2424 + FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) == 2425 2425 + STRTAB_STE_0_CFG_S1_TRANS; 2426 2426 + master->ste_ats_enabled = 2427 2427 + FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) == 2428 2428 + STRTAB_STE_1_EATS_TRANS; 2429 2429 + 2460 2430 for (i = 0; i < master->num_streams; ++i) { 2461 2431 u32 sid = master->streams[i].id; 2462 2432 struct arm_smmu_ste *step = ··· 2495 2451 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)); 2496 2452 } 2497 2453 2498 2498 - static void arm_smmu_enable_ats(struct arm_smmu_master *master, 2499 2499 - struct arm_smmu_domain *smmu_domain) 2454 2454 + static void arm_smmu_enable_ats(struct arm_smmu_master *master) 2500 2455 { 2501 2456 size_t stu; 2502 2457 struct pci_dev *pdev; 2503 2458 struct arm_smmu_device *smmu = master->smmu; 2504 2459 2505 2505 - /* Don't enable ATS at the endpoint if it's not enabled in the STE */ 2506 2506 - if (!master->ats_enabled) 2507 2507 - return; 2508 2508 - 2509 2460 /* Smallest Translation Unit: log2 of the smallest supported granule */ 2510 2461 stu = __ffs(smmu->pgsize_bitmap); 2511 2462 pdev = to_pci_dev(master->dev); 2512 2463 2513 2513 - atomic_inc(&smmu_domain->nr_ats_masters); 2514 2464 /* 2515 2465 * ATC invalidation of PASID 0 causes the entire ATC to be flushed. 2516 2466 */ 2517 2517 - arm_smmu_atc_inv_master(master); 2467 2467 + arm_smmu_atc_inv_master(master, IOMMU_NO_PASID); 2518 2468 if (pci_enable_ats(pdev, stu)) 2519 2469 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); 2520 2520 - } 2521 2521 - 2522 2522 - static void arm_smmu_disable_ats(struct arm_smmu_master *master, 2523 2523 - struct arm_smmu_domain *smmu_domain) 2524 2524 - { 2525 2525 - if (!master->ats_enabled) 2526 2526 - return; 2527 2527 - 2528 2528 - pci_disable_ats(to_pci_dev(master->dev)); 2529 2529 - /* 2530 2530 - * Ensure ATS is disabled at the endpoint before we issue the 2531 2531 - * ATC invalidation via the SMMU. 2532 2532 - */ 2533 2533 - wmb(); 2534 2534 - arm_smmu_atc_inv_master(master); 2535 2535 - atomic_dec(&smmu_domain->nr_ats_masters); 2536 2470 } 2537 2471 2538 2472 static int arm_smmu_enable_pasid(struct arm_smmu_master *master) ··· 2560 2538 pci_disable_pasid(pdev); 2561 2539 } 2562 2540 2563 2563 - static void arm_smmu_detach_dev(struct arm_smmu_master *master) 2541 2541 + static struct arm_smmu_master_domain * 2542 2542 + arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain, 2543 2543 + struct arm_smmu_master *master, 2544 2544 + ioasid_t ssid) 2564 2545 { 2565 2565 - struct iommu_domain *domain = iommu_get_domain_for_dev(master->dev); 2566 2566 - struct arm_smmu_domain *smmu_domain; 2546 2546 + struct arm_smmu_master_domain *master_domain; 2547 2547 + 2548 2548 + lockdep_assert_held(&smmu_domain->devices_lock); 2549 2549 + 2550 2550 + list_for_each_entry(master_domain, &smmu_domain->devices, 2551 2551 + devices_elm) { 2552 2552 + if (master_domain->master == master && 2553 2553 + master_domain->ssid == ssid) 2554 2554 + return master_domain; 2555 2555 + } 2556 2556 + return NULL; 2557 2557 + } 2558 2558 + 2559 2559 + /* 2560 2560 + * If the domain uses the smmu_domain->devices list return the arm_smmu_domain 2561 2561 + * structure, otherwise NULL. These domains track attached devices so they can 2562 2562 + * issue invalidations. 2563 2563 + */ 2564 2564 + static struct arm_smmu_domain * 2565 2565 + to_smmu_domain_devices(struct iommu_domain *domain) 2566 2566 + { 2567 2567 + /* The domain can be NULL only when processing the first attach */ 2568 2568 + if (!domain) 2569 2569 + return NULL; 2570 2570 + if ((domain->type & __IOMMU_DOMAIN_PAGING) || 2571 2571 + domain->type == IOMMU_DOMAIN_SVA) 2572 2572 + return to_smmu_domain(domain); 2573 2573 + return NULL; 2574 2574 + } 2575 2575 + 2576 2576 + static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, 2577 2577 + struct iommu_domain *domain, 2578 2578 + ioasid_t ssid) 2579 2579 + { 2580 2580 + struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain); 2581 2581 + struct arm_smmu_master_domain *master_domain; 2567 2582 unsigned long flags; 2568 2583 2569 2569 - if (!domain || !(domain->type & __IOMMU_DOMAIN_PAGING)) 2584 2584 + if (!smmu_domain) 2570 2585 return; 2571 2586 2572 2572 - smmu_domain = to_smmu_domain(domain); 2573 2573 - arm_smmu_disable_ats(master, smmu_domain); 2574 2574 - 2575 2587 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2576 2576 - list_del_init(&master->domain_head); 2588 2588 + master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid); 2589 2589 + if (master_domain) { 2590 2590 + list_del(&master_domain->devices_elm); 2591 2591 + kfree(master_domain); 2592 2592 + if (master->ats_enabled) 2593 2593 + atomic_dec(&smmu_domain->nr_ats_masters); 2594 2594 + } 2577 2595 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2596 2596 + } 2578 2597 2579 2579 - master->ats_enabled = false; 2598 2598 + struct arm_smmu_attach_state { 2599 2599 + /* Inputs */ 2600 2600 + struct iommu_domain *old_domain; 2601 2601 + struct arm_smmu_master *master; 2602 2602 + bool cd_needs_ats; 2603 2603 + ioasid_t ssid; 2604 2604 + /* Resulting state */ 2605 2605 + bool ats_enabled; 2606 2606 + }; 2607 2607 + 2608 2608 + /* 2609 2609 + * Start the sequence to attach a domain to a master. The sequence contains three 2610 2610 + * steps: 2611 2611 + * arm_smmu_attach_prepare() 2612 2612 + * arm_smmu_install_ste_for_dev() 2613 2613 + * arm_smmu_attach_commit() 2614 2614 + * 2615 2615 + * If prepare succeeds then the sequence must be completed. The STE installed 2616 2616 + * must set the STE.EATS field according to state.ats_enabled. 2617 2617 + * 2618 2618 + * If the device supports ATS then this determines if EATS should be enabled 2619 2619 + * in the STE, and starts sequencing EATS disable if required. 2620 2620 + * 2621 2621 + * The change of the EATS in the STE and the PCI ATS config space is managed by 2622 2622 + * this sequence to be in the right order so that if PCI ATS is enabled then 2623 2623 + * STE.ETAS is enabled. 2624 2624 + * 2625 2625 + * new_domain can be a non-paging domain. In this case ATS will not be enabled, 2626 2626 + * and invalidations won't be tracked. 2627 2627 + */ 2628 2628 + static int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, 2629 2629 + struct iommu_domain *new_domain) 2630 2630 + { 2631 2631 + struct arm_smmu_master *master = state->master; 2632 2632 + struct arm_smmu_master_domain *master_domain; 2633 2633 + struct arm_smmu_domain *smmu_domain = 2634 2634 + to_smmu_domain_devices(new_domain); 2635 2635 + unsigned long flags; 2636 2636 + 2637 2637 + /* 2638 2638 + * arm_smmu_share_asid() must not see two domains pointing to the same 2639 2639 + * arm_smmu_master_domain contents otherwise it could randomly write one 2640 2640 + * or the other to the CD. 2641 2641 + */ 2642 2642 + lockdep_assert_held(&arm_smmu_asid_lock); 2643 2643 + 2644 2644 + if (smmu_domain || state->cd_needs_ats) { 2645 2645 + /* 2646 2646 + * The SMMU does not support enabling ATS with bypass/abort. 2647 2647 + * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS 2648 2648 + * Translation Requests and Translated transactions are denied 2649 2649 + * as though ATS is disabled for the stream (STE.EATS == 0b00), 2650 2650 + * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events 2651 2651 + * (IHI0070Ea 5.2 Stream Table Entry). Thus ATS can only be 2652 2652 + * enabled if we have arm_smmu_domain, those always have page 2653 2653 + * tables. 2654 2654 + */ 2655 2655 + state->ats_enabled = arm_smmu_ats_supported(master); 2656 2656 + } 2657 2657 + 2658 2658 + if (smmu_domain) { 2659 2659 + master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL); 2660 2660 + if (!master_domain) 2661 2661 + return -ENOMEM; 2662 2662 + master_domain->master = master; 2663 2663 + master_domain->ssid = state->ssid; 2664 2664 + 2665 2665 + /* 2666 2666 + * During prepare we want the current smmu_domain and new 2667 2667 + * smmu_domain to be in the devices list before we change any 2668 2668 + * HW. This ensures that both domains will send ATS 2669 2669 + * invalidations to the master until we are done. 2670 2670 + * 2671 2671 + * It is tempting to make this list only track masters that are 2672 2672 + * using ATS, but arm_smmu_share_asid() also uses this to change 2673 2673 + * the ASID of a domain, unrelated to ATS. 2674 2674 + * 2675 2675 + * Notice if we are re-attaching the same domain then the list 2676 2676 + * will have two identical entries and commit will remove only 2677 2677 + * one of them. 2678 2678 + */ 2679 2679 + spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2680 2680 + if (state->ats_enabled) 2681 2681 + atomic_inc(&smmu_domain->nr_ats_masters); 2682 2682 + list_add(&master_domain->devices_elm, &smmu_domain->devices); 2683 2683 + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2684 2684 + } 2685 2685 + 2686 2686 + if (!state->ats_enabled && master->ats_enabled) { 2687 2687 + pci_disable_ats(to_pci_dev(master->dev)); 2688 2688 + /* 2689 2689 + * This is probably overkill, but the config write for disabling 2690 2690 + * ATS should complete before the STE is configured to generate 2691 2691 + * UR to avoid AER noise. 2692 2692 + */ 2693 2693 + wmb(); 2694 2694 + } 2695 2695 + return 0; 2696 2696 + } 2697 2697 + 2698 2698 + /* 2699 2699 + * Commit is done after the STE/CD are configured with the EATS setting. It 2700 2700 + * completes synchronizing the PCI device's ATC and finishes manipulating the 2701 2701 + * smmu_domain->devices list. 2702 2702 + */ 2703 2703 + static void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) 2704 2704 + { 2705 2705 + struct arm_smmu_master *master = state->master; 2706 2706 + 2707 2707 + lockdep_assert_held(&arm_smmu_asid_lock); 2708 2708 + 2709 2709 + if (state->ats_enabled && !master->ats_enabled) { 2710 2710 + arm_smmu_enable_ats(master); 2711 2711 + } else if (state->ats_enabled && master->ats_enabled) { 2712 2712 + /* 2713 2713 + * The translation has changed, flush the ATC. At this point the 2714 2714 + * SMMU is translating for the new domain and both the old&new 2715 2715 + * domain will issue invalidations. 2716 2716 + */ 2717 2717 + arm_smmu_atc_inv_master(master, state->ssid); 2718 2718 + } else if (!state->ats_enabled && master->ats_enabled) { 2719 2719 + /* ATS is being switched off, invalidate the entire ATC */ 2720 2720 + arm_smmu_atc_inv_master(master, IOMMU_NO_PASID); 2721 2721 + } 2722 2722 + master->ats_enabled = state->ats_enabled; 2723 2723 + 2724 2724 + arm_smmu_remove_master_domain(master, state->old_domain, state->ssid); 2580 2725 } 2581 2726 2582 2727 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 2583 2728 { 2584 2729 int ret = 0; 2585 2585 - unsigned long flags; 2586 2730 struct arm_smmu_ste target; 2587 2731 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2588 2732 struct arm_smmu_device *smmu; 2589 2733 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2734 2734 + struct arm_smmu_attach_state state = { 2735 2735 + .old_domain = iommu_get_domain_for_dev(dev), 2736 2736 + .ssid = IOMMU_NO_PASID, 2737 2737 + }; 2590 2738 struct arm_smmu_master *master; 2591 2739 struct arm_smmu_cd *cdptr; 2592 2740 2593 2741 if (!fwspec) 2594 2742 return -ENOENT; 2595 2743 2596 2596 - master = dev_iommu_priv_get(dev); 2744 2744 + state.master = master = dev_iommu_priv_get(dev); 2597 2745 smmu = master->smmu; 2598 2598 - 2599 2599 - /* 2600 2600 - * Checking that SVA is disabled ensures that this device isn't bound to 2601 2601 - * any mm, and can be safely detached from its old domain. Bonds cannot 2602 2602 - * be removed concurrently since we're holding the group mutex. 2603 2603 - */ 2604 2604 - if (arm_smmu_master_sva_enabled(master)) { 2605 2605 - dev_err(dev, "cannot attach - SVA enabled\n"); 2606 2606 - return -EBUSY; 2607 2607 - } 2608 2746 2609 2747 mutex_lock(&smmu_domain->init_mutex); 2610 2748 2611 2749 if (!smmu_domain->smmu) { 2612 2612 - ret = arm_smmu_domain_finalise(smmu_domain, smmu); 2750 2750 + ret = arm_smmu_domain_finalise(smmu_domain, smmu, 0); 2613 2751 } else if (smmu_domain->smmu != smmu) 2614 2752 ret = -EINVAL; 2615 2753 ··· 2781 2599 cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); 2782 2600 if (!cdptr) 2783 2601 return -ENOMEM; 2784 2784 - } 2602 2602 + } else if (arm_smmu_ssids_in_use(&master->cd_table)) 2603 2603 + return -EBUSY; 2785 2604 2786 2605 /* 2787 2606 * Prevent arm_smmu_share_asid() from trying to change the ASID ··· 2792 2609 */ 2793 2610 mutex_lock(&arm_smmu_asid_lock); 2794 2611 2795 2795 - arm_smmu_detach_dev(master); 2796 2796 - 2797 2797 - master->ats_enabled = arm_smmu_ats_supported(master); 2798 2798 - 2799 2799 - spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2800 2800 - list_add(&master->domain_head, &smmu_domain->devices); 2801 2801 - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2612 2612 + ret = arm_smmu_attach_prepare(&state, domain); 2613 2613 + if (ret) { 2614 2614 + mutex_unlock(&arm_smmu_asid_lock); 2615 2615 + return ret; 2616 2616 + } 2802 2617 2803 2618 switch (smmu_domain->stage) { 2804 2619 case ARM_SMMU_DOMAIN_S1: { ··· 2805 2624 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 2806 2625 arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, 2807 2626 &target_cd); 2808 2808 - arm_smmu_make_cdtable_ste(&target, master); 2627 2627 + arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled, 2628 2628 + STRTAB_STE_1_S1DSS_SSID0); 2809 2629 arm_smmu_install_ste_for_dev(master, &target); 2810 2630 break; 2811 2631 } 2812 2632 case ARM_SMMU_DOMAIN_S2: 2813 2813 - arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); 2633 2633 + arm_smmu_make_s2_domain_ste(&target, master, smmu_domain, 2634 2634 + state.ats_enabled); 2814 2635 arm_smmu_install_ste_for_dev(master, &target); 2815 2636 arm_smmu_clear_cd(master, IOMMU_NO_PASID); 2816 2637 break; 2817 2638 } 2818 2639 2819 2819 - arm_smmu_enable_ats(master, smmu_domain); 2640 2640 + arm_smmu_attach_commit(&state); 2820 2641 mutex_unlock(&arm_smmu_asid_lock); 2821 2642 return 0; 2822 2643 } 2823 2644 2824 2824 - static int arm_smmu_attach_dev_ste(struct device *dev, 2825 2825 - struct arm_smmu_ste *ste) 2645 2645 + static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain, 2646 2646 + struct device *dev, ioasid_t id) 2647 2647 + { 2648 2648 + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2649 2649 + struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2650 2650 + struct arm_smmu_device *smmu = master->smmu; 2651 2651 + struct arm_smmu_cd target_cd; 2652 2652 + int ret = 0; 2653 2653 + 2654 2654 + mutex_lock(&smmu_domain->init_mutex); 2655 2655 + if (!smmu_domain->smmu) 2656 2656 + ret = arm_smmu_domain_finalise(smmu_domain, smmu, 0); 2657 2657 + else if (smmu_domain->smmu != smmu) 2658 2658 + ret = -EINVAL; 2659 2659 + mutex_unlock(&smmu_domain->init_mutex); 2660 2660 + if (ret) 2661 2661 + return ret; 2662 2662 + 2663 2663 + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) 2664 2664 + return -EINVAL; 2665 2665 + 2666 2666 + /* 2667 2667 + * We can read cd.asid outside the lock because arm_smmu_set_pasid() 2668 2668 + * will fix it 2669 2669 + */ 2670 2670 + arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 2671 2671 + return arm_smmu_set_pasid(master, to_smmu_domain(domain), id, 2672 2672 + &target_cd); 2673 2673 + } 2674 2674 + 2675 2675 + static void arm_smmu_update_ste(struct arm_smmu_master *master, 2676 2676 + struct iommu_domain *sid_domain, 2677 2677 + bool ats_enabled) 2678 2678 + { 2679 2679 + unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE; 2680 2680 + struct arm_smmu_ste ste; 2681 2681 + 2682 2682 + if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled) 2683 2683 + return; 2684 2684 + 2685 2685 + if (sid_domain->type == IOMMU_DOMAIN_IDENTITY) 2686 2686 + s1dss = STRTAB_STE_1_S1DSS_BYPASS; 2687 2687 + else 2688 2688 + WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED); 2689 2689 + 2690 2690 + /* 2691 2691 + * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior 2692 2692 + * using s1dss if necessary. If the cd_table is already installed then 2693 2693 + * the S1DSS is correct and this will just update the EATS. Otherwise it 2694 2694 + * installs the entire thing. This will be hitless. 2695 2695 + */ 2696 2696 + arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss); 2697 2697 + arm_smmu_install_ste_for_dev(master, &ste); 2698 2698 + } 2699 2699 + 2700 2700 + int arm_smmu_set_pasid(struct arm_smmu_master *master, 2701 2701 + struct arm_smmu_domain *smmu_domain, ioasid_t pasid, 2702 2702 + struct arm_smmu_cd *cd) 2703 2703 + { 2704 2704 + struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev); 2705 2705 + struct arm_smmu_attach_state state = { 2706 2706 + .master = master, 2707 2707 + /* 2708 2708 + * For now the core code prevents calling this when a domain is 2709 2709 + * already attached, no need to set old_domain. 2710 2710 + */ 2711 2711 + .ssid = pasid, 2712 2712 + }; 2713 2713 + struct arm_smmu_cd *cdptr; 2714 2714 + int ret; 2715 2715 + 2716 2716 + /* The core code validates pasid */ 2717 2717 + 2718 2718 + if (smmu_domain->smmu != master->smmu) 2719 2719 + return -EINVAL; 2720 2720 + 2721 2721 + if (!master->cd_table.in_ste && 2722 2722 + sid_domain->type != IOMMU_DOMAIN_IDENTITY && 2723 2723 + sid_domain->type != IOMMU_DOMAIN_BLOCKED) 2724 2724 + return -EINVAL; 2725 2725 + 2726 2726 + cdptr = arm_smmu_alloc_cd_ptr(master, pasid); 2727 2727 + if (!cdptr) 2728 2728 + return -ENOMEM; 2729 2729 + 2730 2730 + mutex_lock(&arm_smmu_asid_lock); 2731 2731 + ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain); 2732 2732 + if (ret) 2733 2733 + goto out_unlock; 2734 2734 + 2735 2735 + /* 2736 2736 + * We don't want to obtain to the asid_lock too early, so fix up the 2737 2737 + * caller set ASID under the lock in case it changed. 2738 2738 + */ 2739 2739 + cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID); 2740 2740 + cd->data[0] |= cpu_to_le64( 2741 2741 + FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid)); 2742 2742 + 2743 2743 + arm_smmu_write_cd_entry(master, pasid, cdptr, cd); 2744 2744 + arm_smmu_update_ste(master, sid_domain, state.ats_enabled); 2745 2745 + 2746 2746 + arm_smmu_attach_commit(&state); 2747 2747 + 2748 2748 + out_unlock: 2749 2749 + mutex_unlock(&arm_smmu_asid_lock); 2750 2750 + return ret; 2751 2751 + } 2752 2752 + 2753 2753 + static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid, 2754 2754 + struct iommu_domain *domain) 2826 2755 { 2827 2756 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2757 2757 + struct arm_smmu_domain *smmu_domain; 2828 2758 2829 2829 - if (arm_smmu_master_sva_enabled(master)) 2830 2830 - return -EBUSY; 2759 2759 + smmu_domain = to_smmu_domain(domain); 2760 2760 + 2761 2761 + mutex_lock(&arm_smmu_asid_lock); 2762 2762 + arm_smmu_clear_cd(master, pasid); 2763 2763 + if (master->ats_enabled) 2764 2764 + arm_smmu_atc_inv_master(master, pasid); 2765 2765 + arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid); 2766 2766 + mutex_unlock(&arm_smmu_asid_lock); 2767 2767 + 2768 2768 + /* 2769 2769 + * When the last user of the CD table goes away downgrade the STE back 2770 2770 + * to a non-cd_table one. 2771 2771 + */ 2772 2772 + if (!arm_smmu_ssids_in_use(&master->cd_table)) { 2773 2773 + struct iommu_domain *sid_domain = 2774 2774 + iommu_get_domain_for_dev(master->dev); 2775 2775 + 2776 2776 + if (sid_domain->type == IOMMU_DOMAIN_IDENTITY || 2777 2777 + sid_domain->type == IOMMU_DOMAIN_BLOCKED) 2778 2778 + sid_domain->ops->attach_dev(sid_domain, dev); 2779 2779 + } 2780 2780 + } 2781 2781 + 2782 2782 + static void arm_smmu_attach_dev_ste(struct iommu_domain *domain, 2783 2783 + struct device *dev, 2784 2784 + struct arm_smmu_ste *ste, 2785 2785 + unsigned int s1dss) 2786 2786 + { 2787 2787 + struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2788 2788 + struct arm_smmu_attach_state state = { 2789 2789 + .master = master, 2790 2790 + .old_domain = iommu_get_domain_for_dev(dev), 2791 2791 + .ssid = IOMMU_NO_PASID, 2792 2792 + }; 2831 2793 2832 2794 /* 2833 2795 * Do not allow any ASID to be changed while are working on the STE, ··· 2979 2655 mutex_lock(&arm_smmu_asid_lock); 2980 2656 2981 2657 /* 2982 2982 - * The SMMU does not support enabling ATS with bypass/abort. When the 2983 2983 - * STE is in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests 2984 2984 - * and Translated transactions are denied as though ATS is disabled for 2985 2985 - * the stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and 2986 2986 - * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). 2658 2658 + * If the CD table is not in use we can use the provided STE, otherwise 2659 2659 + * we use a cdtable STE with the provided S1DSS. 2987 2660 */ 2988 2988 - arm_smmu_detach_dev(master); 2989 2989 - 2661 2661 + if (arm_smmu_ssids_in_use(&master->cd_table)) { 2662 2662 + /* 2663 2663 + * If a CD table has to be present then we need to run with ATS 2664 2664 + * on even though the RID will fail ATS queries with UR. This is 2665 2665 + * because we have no idea what the PASID's need. 2666 2666 + */ 2667 2667 + state.cd_needs_ats = true; 2668 2668 + arm_smmu_attach_prepare(&state, domain); 2669 2669 + arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss); 2670 2670 + } else { 2671 2671 + arm_smmu_attach_prepare(&state, domain); 2672 2672 + } 2990 2673 arm_smmu_install_ste_for_dev(master, ste); 2674 2674 + arm_smmu_attach_commit(&state); 2991 2675 mutex_unlock(&arm_smmu_asid_lock); 2992 2676 2993 2677 /* ··· 3004 2672 * descriptor from arm_smmu_share_asid(). 3005 2673 */ 3006 2674 arm_smmu_clear_cd(master, IOMMU_NO_PASID); 3007 3007 - return 0; 3008 2675 } 3009 2676 3010 2677 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, ··· 3013 2682 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 3014 2683 3015 2684 arm_smmu_make_bypass_ste(master->smmu, &ste); 3016 3016 - return arm_smmu_attach_dev_ste(dev, &ste); 2685 2685 + arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS); 2686 2686 + return 0; 3017 2687 } 3018 2688 3019 2689 static const struct iommu_domain_ops arm_smmu_identity_ops = { ··· 3032 2700 struct arm_smmu_ste ste; 3033 2701 3034 2702 arm_smmu_make_abort_ste(&ste); 3035 3035 - return arm_smmu_attach_dev_ste(dev, &ste); 2703 2703 + arm_smmu_attach_dev_ste(domain, dev, &ste, 2704 2704 + STRTAB_STE_1_S1DSS_TERMINATE); 2705 2705 + return 0; 3036 2706 } 3037 2707 3038 2708 static const struct iommu_domain_ops arm_smmu_blocked_ops = { ··· 3045 2711 .type = IOMMU_DOMAIN_BLOCKED, 3046 2712 .ops = &arm_smmu_blocked_ops, 3047 2713 }; 2714 2714 + 2715 2715 + static struct iommu_domain * 2716 2716 + arm_smmu_domain_alloc_user(struct device *dev, u32 flags, 2717 2717 + struct iommu_domain *parent, 2718 2718 + const struct iommu_user_data *user_data) 2719 2719 + { 2720 2720 + struct arm_smmu_master *master = dev_iommu_priv_get(dev); 2721 2721 + const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING; 2722 2722 + struct arm_smmu_domain *smmu_domain; 2723 2723 + int ret; 2724 2724 + 2725 2725 + if (flags & ~PAGING_FLAGS) 2726 2726 + return ERR_PTR(-EOPNOTSUPP); 2727 2727 + if (parent || user_data) 2728 2728 + return ERR_PTR(-EOPNOTSUPP); 2729 2729 + 2730 2730 + smmu_domain = arm_smmu_domain_alloc(); 2731 2731 + if (!smmu_domain) 2732 2732 + return ERR_PTR(-ENOMEM); 2733 2733 + 2734 2734 + smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED; 2735 2735 + smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops; 2736 2736 + ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, flags); 2737 2737 + if (ret) 2738 2738 + goto err_free; 2739 2739 + return &smmu_domain->domain; 2740 2740 + 2741 2741 + err_free: 2742 2742 + kfree(smmu_domain); 2743 2743 + return ERR_PTR(ret); 2744 2744 + } 3048 2745 3049 2746 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 3050 2747 phys_addr_t paddr, size_t pgsize, size_t pgcount, ··· 3247 2882 kfree(master->streams); 3248 2883 } 3249 2884 3250 3250 - static struct iommu_ops arm_smmu_ops; 3251 3251 - 3252 2885 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 3253 2886 { 3254 2887 int ret; ··· 3267 2904 3268 2905 master->dev = dev; 3269 2906 master->smmu = smmu; 3270 3270 - INIT_LIST_HEAD(&master->bonds); 3271 3271 - INIT_LIST_HEAD(&master->domain_head); 3272 2907 dev_iommu_priv_set(dev, master); 3273 2908 3274 2909 ret = arm_smmu_insert_master(smmu, master); ··· 3320 2959 if (master->cd_table.cdtab) 3321 2960 arm_smmu_free_cd_tables(master); 3322 2961 kfree(master); 2962 2962 + } 2963 2963 + 2964 2964 + static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain, 2965 2965 + unsigned long iova, size_t size, 2966 2966 + unsigned long flags, 2967 2967 + struct iommu_dirty_bitmap *dirty) 2968 2968 + { 2969 2969 + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 2970 2970 + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 2971 2971 + 2972 2972 + return ops->read_and_clear_dirty(ops, iova, size, flags, dirty); 2973 2973 + } 2974 2974 + 2975 2975 + static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain, 2976 2976 + bool enabled) 2977 2977 + { 2978 2978 + /* 2979 2979 + * Always enabled and the dirty bitmap is cleared prior to 2980 2980 + * set_dirty_tracking(). 2981 2981 + */ 2982 2982 + return 0; 3323 2983 } 3324 2984 3325 2985 static struct iommu_group *arm_smmu_device_group(struct device *dev) ··· 3469 3087 return 0; 3470 3088 } 3471 3089 3472 3472 - static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid, 3473 3473 - struct iommu_domain *domain) 3474 3474 - { 3475 3475 - arm_smmu_sva_remove_dev_pasid(domain, dev, pasid); 3476 3476 - } 3477 3477 - 3478 3090 static struct iommu_ops arm_smmu_ops = { 3479 3091 .identity_domain = &arm_smmu_identity_domain, 3480 3092 .blocked_domain = &arm_smmu_blocked_domain, 3481 3093 .capable = arm_smmu_capable, 3482 3482 - .domain_alloc = arm_smmu_domain_alloc, 3483 3094 .domain_alloc_paging = arm_smmu_domain_alloc_paging, 3095 3095 + .domain_alloc_sva = arm_smmu_sva_domain_alloc, 3096 3096 + .domain_alloc_user = arm_smmu_domain_alloc_user, 3484 3097 .probe_device = arm_smmu_probe_device, 3485 3098 .release_device = arm_smmu_release_device, 3486 3099 .device_group = arm_smmu_device_group, ··· 3490 3113 .owner = THIS_MODULE, 3491 3114 .default_domain_ops = &(const struct iommu_domain_ops) { 3492 3115 .attach_dev = arm_smmu_attach_dev, 3116 3116 + .set_dev_pasid = arm_smmu_s1_set_dev_pasid, 3493 3117 .map_pages = arm_smmu_map_pages, 3494 3118 .unmap_pages = arm_smmu_unmap_pages, 3495 3119 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 3496 3120 .iotlb_sync = arm_smmu_iotlb_sync, 3497 3121 .iova_to_phys = arm_smmu_iova_to_phys, 3498 3122 .enable_nesting = arm_smmu_enable_nesting, 3499 3499 - .free = arm_smmu_domain_free, 3123 3123 + .free = arm_smmu_domain_free_paging, 3500 3124 } 3125 3125 + }; 3126 3126 + 3127 3127 + static struct iommu_dirty_ops arm_smmu_dirty_ops = { 3128 3128 + .read_and_clear_dirty = arm_smmu_read_and_clear_dirty, 3129 3129 + .set_dirty_tracking = arm_smmu_set_dirty_tracking, 3501 3130 }; 3502 3131 3503 3132 /* Probing and initialisation functions */ ··· 3604 3221 PRIQ_ENT_DWORDS, "priq"); 3605 3222 } 3606 3223 3607 3607 - static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) 3608 3608 - { 3609 3609 - unsigned int i; 3610 3610 - struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; 3611 3611 - void *strtab = smmu->strtab_cfg.strtab; 3612 3612 - 3613 3613 - cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents, 3614 3614 - sizeof(*cfg->l1_desc), GFP_KERNEL); 3615 3615 - if (!cfg->l1_desc) 3616 3616 - return -ENOMEM; 3617 3617 - 3618 3618 - for (i = 0; i < cfg->num_l1_ents; ++i) { 3619 3619 - arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]); 3620 3620 - strtab += STRTAB_L1_DESC_DWORDS << 3; 3621 3621 - } 3622 3622 - 3623 3623 - return 0; 3624 3624 - } 3625 3625 - 3626 3224 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) 3627 3225 { 3628 3226 void *strtab; ··· 3639 3275 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT); 3640 3276 cfg->strtab_base_cfg = reg; 3641 3277 3642 3642 - return arm_smmu_init_l1_strtab(smmu); 3278 3278 + cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents, 3279 3279 + sizeof(*cfg->l1_desc), GFP_KERNEL); 3280 3280 + if (!cfg->l1_desc) 3281 3281 + return -ENOMEM; 3282 3282 + 3283 3283 + return 0; 3643 3284 } 3644 3285 3645 3286 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) ··· 4067 3698 } 4068 3699 } 4069 3700 3701 3701 + static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) 3702 3702 + { 3703 3703 + u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD); 3704 3704 + u32 hw_features = 0; 3705 3705 + 3706 3706 + switch (FIELD_GET(IDR0_HTTU, reg)) { 3707 3707 + case IDR0_HTTU_ACCESS_DIRTY: 3708 3708 + hw_features |= ARM_SMMU_FEAT_HD; 3709 3709 + fallthrough; 3710 3710 + case IDR0_HTTU_ACCESS: 3711 3711 + hw_features |= ARM_SMMU_FEAT_HA; 3712 3712 + } 3713 3713 + 3714 3714 + if (smmu->dev->of_node) 3715 3715 + smmu->features |= hw_features; 3716 3716 + else if (hw_features != fw_features) 3717 3717 + /* ACPI IORT sets the HTTU bits */ 3718 3718 + dev_warn(smmu->dev, 3719 3719 + "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n", 3720 3720 + hw_features, fw_features); 3721 3721 + } 3722 3722 + 4070 3723 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) 4071 3724 { 4072 3725 u32 reg; ··· 4148 3757 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)) 4149 3758 smmu->features |= ARM_SMMU_FEAT_E2H; 4150 3759 } 3760 3760 + 3761 3761 + arm_smmu_get_httu(smmu, reg); 4151 3762 4152 3763 /* 4153 3764 * The coherency feature as set by FW is used in preference to the ID ··· 4345 3952 4346 3953 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) 4347 3954 smmu->features |= ARM_SMMU_FEAT_COHERENCY; 3955 3955 + 3956 3956 + switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) { 3957 3957 + case IDR0_HTTU_ACCESS_DIRTY: 3958 3958 + smmu->features |= ARM_SMMU_FEAT_HD; 3959 3959 + fallthrough; 3960 3960 + case IDR0_HTTU_ACCESS: 3961 3961 + smmu->features |= ARM_SMMU_FEAT_HA; 3962 3962 + } 4348 3963 4349 3964 return 0; 4350 3965 }

+40 -23

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

reviewed

··· 33 33 #define IDR0_ASID16 (1 << 12) 34 34 #define IDR0_ATS (1 << 10) 35 35 #define IDR0_HYP (1 << 9) 36 36 + #define IDR0_HTTU GENMASK(7, 6) 37 37 + #define IDR0_HTTU_ACCESS 1 38 38 + #define IDR0_HTTU_ACCESS_DIRTY 2 36 39 #define IDR0_COHACC (1 << 4) 37 40 #define IDR0_TTF GENMASK(3, 2) 38 41 #define IDR0_TTF_AARCH64 2 ··· 303 300 304 301 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) 305 302 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) 303 303 + 304 304 + #define CTXDESC_CD_0_TCR_HA (1UL << 43) 305 305 + #define CTXDESC_CD_0_TCR_HD (1UL << 42) 306 306 307 307 #define CTXDESC_CD_0_AA64 (1UL << 41) 308 308 #define CTXDESC_CD_0_S (1UL << 44) ··· 585 579 586 580 /* High-level stream table and context descriptor structures */ 587 581 struct arm_smmu_strtab_l1_desc { 588 588 - u8 span; 589 589 - 590 582 struct arm_smmu_ste *l2ptr; 591 591 - dma_addr_t l2ptr_dma; 592 583 }; 593 584 594 585 struct arm_smmu_ctx_desc { 595 586 u16 asid; 596 596 - 597 597 - refcount_t refs; 598 598 - struct mm_struct *mm; 599 587 }; 600 588 601 589 struct arm_smmu_l1_ctx_desc { ··· 602 602 dma_addr_t cdtab_dma; 603 603 struct arm_smmu_l1_ctx_desc *l1_desc; 604 604 unsigned int num_l1_ents; 605 605 + unsigned int used_ssids; 606 606 + u8 in_ste; 605 607 u8 s1fmt; 606 608 /* log2 of the maximum number of CDs supported by this table */ 607 609 u8 s1cdmax; 608 610 }; 611 611 + 612 612 + /* True if the cd table has SSIDS > 0 in use. */ 613 613 + static inline bool arm_smmu_ssids_in_use(struct arm_smmu_ctx_desc_cfg *cd_table) 614 614 + { 615 615 + return cd_table->used_ssids; 616 616 + } 609 617 610 618 struct arm_smmu_s2_cfg { 611 619 u16 vmid; ··· 656 648 #define ARM_SMMU_FEAT_E2H (1 << 18) 657 649 #define ARM_SMMU_FEAT_NESTING (1 << 19) 658 650 #define ARM_SMMU_FEAT_ATTR_TYPES_OVR (1 << 20) 651 651 + #define ARM_SMMU_FEAT_HA (1 << 21) 652 652 + #define ARM_SMMU_FEAT_HD (1 << 22) 659 653 u32 features; 660 654 661 655 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) ··· 706 696 struct arm_smmu_master { 707 697 struct arm_smmu_device *smmu; 708 698 struct device *dev; 709 709 - struct list_head domain_head; 710 699 struct arm_smmu_stream *streams; 711 700 /* Locked by the iommu core using the group mutex */ 712 701 struct arm_smmu_ctx_desc_cfg cd_table; 713 702 unsigned int num_streams; 714 714 - bool ats_enabled; 703 703 + bool ats_enabled : 1; 704 704 + bool ste_ats_enabled : 1; 715 705 bool stall_enabled; 716 706 bool sva_enabled; 717 707 bool iopf_enabled; 718 718 - struct list_head bonds; 719 708 unsigned int ssid_bits; 720 709 }; 721 710 ··· 739 730 740 731 struct iommu_domain domain; 741 732 733 733 + /* List of struct arm_smmu_master_domain */ 742 734 struct list_head devices; 743 735 spinlock_t devices_lock; 744 736 745 745 - struct list_head mmu_notifiers; 737 737 + struct mmu_notifier mmu_notifier; 746 738 }; 747 739 748 740 /* The following are exposed for testing purposes. */ ··· 767 757 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, 768 758 struct arm_smmu_ste *target); 769 759 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, 770 770 - struct arm_smmu_master *master); 760 760 + struct arm_smmu_master *master, bool ats_enabled, 761 761 + unsigned int s1dss); 771 762 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, 772 763 struct arm_smmu_master *master, 773 773 - struct arm_smmu_domain *smmu_domain); 764 764 + struct arm_smmu_domain *smmu_domain, 765 765 + bool ats_enabled); 774 766 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, 775 767 struct arm_smmu_master *master, struct mm_struct *mm, 776 768 u16 asid); 777 769 #endif 770 770 + 771 771 + struct arm_smmu_master_domain { 772 772 + struct list_head devices_elm; 773 773 + struct arm_smmu_master *master; 774 774 + ioasid_t ssid; 775 775 + }; 778 776 779 777 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 780 778 { ··· 792 774 extern struct xarray arm_smmu_asid_xa; 793 775 extern struct mutex arm_smmu_asid_lock; 794 776 777 777 + struct arm_smmu_domain *arm_smmu_domain_alloc(void); 778 778 + 795 779 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); 796 780 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, 797 781 u32 ssid); 798 798 - struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, 799 799 - u32 ssid); 800 782 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, 801 783 struct arm_smmu_master *master, 802 784 struct arm_smmu_domain *smmu_domain); ··· 804 786 struct arm_smmu_cd *cdptr, 805 787 const struct arm_smmu_cd *target); 806 788 789 789 + int arm_smmu_set_pasid(struct arm_smmu_master *master, 790 790 + struct arm_smmu_domain *smmu_domain, ioasid_t pasid, 791 791 + struct arm_smmu_cd *cd); 792 792 + 807 793 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); 808 794 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, 809 795 size_t granule, bool leaf, 810 796 struct arm_smmu_domain *smmu_domain); 811 811 - bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd); 812 812 - int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, 797 797 + int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, 813 798 unsigned long iova, size_t size); 814 799 815 800 #ifdef CONFIG_ARM_SMMU_V3_SVA ··· 823 802 int arm_smmu_master_disable_sva(struct arm_smmu_master *master); 824 803 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master); 825 804 void arm_smmu_sva_notifier_synchronize(void); 826 826 - struct iommu_domain *arm_smmu_sva_domain_alloc(void); 827 827 - void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 828 828 - struct device *dev, ioasid_t id); 805 805 + struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, 806 806 + struct mm_struct *mm); 829 807 #else /* CONFIG_ARM_SMMU_V3_SVA */ 830 808 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 831 809 { ··· 858 838 859 839 static inline void arm_smmu_sva_notifier_synchronize(void) {} 860 840 861 861 - static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void) 862 862 - { 863 863 - return NULL; 864 864 - } 841 841 + #define arm_smmu_sva_domain_alloc NULL 865 842 866 843 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, 867 844 struct device *dev,

+1 -1

drivers/iommu/arm/arm-smmu/arm-smmu-nvidia.c

reviewed

··· 200 200 void __iomem *cb_base = nvidia_smmu_page(smmu, inst, smmu->numpage + idx); 201 201 202 202 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); 203 203 - if (!(fsr & ARM_SMMU_FSR_FAULT)) 203 203 + if (!(fsr & ARM_SMMU_CB_FSR_FAULT)) 204 204 return IRQ_NONE; 205 205 206 206 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);

+22 -61

drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c

reviewed

··· 141 141 writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); 142 142 143 143 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 144 144 - if ((fsr & ARM_SMMU_FSR_FAULT) && (fsr & ARM_SMMU_FSR_SS)) { 144 144 + if ((fsr & ARM_SMMU_CB_FSR_FAULT) && (fsr & ARM_SMMU_CB_FSR_SS)) { 145 145 u32 sctlr_orig, sctlr; 146 146 147 147 /* ··· 298 298 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); 299 299 300 300 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 301 301 - if (fsr & ARM_SMMU_FSR_FAULT) { 301 301 + if (fsr & ARM_SMMU_CB_FSR_FAULT) { 302 302 /* Clear pending interrupts */ 303 303 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 304 304 ··· 306 306 * TBU halt takes care of resuming any stalled transcation. 307 307 * Kept it here for completeness sake. 308 308 */ 309 309 - if (fsr & ARM_SMMU_FSR_SS) 309 309 + if (fsr & ARM_SMMU_CB_FSR_SS) 310 310 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 311 311 ARM_SMMU_RESUME_TERMINATE); 312 312 } ··· 320 320 phys = qcom_tbu_trigger_atos(smmu_domain, tbu, iova, sid); 321 321 322 322 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 323 323 - if (fsr & ARM_SMMU_FSR_FAULT) { 323 323 + if (fsr & ARM_SMMU_CB_FSR_FAULT) { 324 324 /* Clear pending interrupts */ 325 325 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 326 326 327 327 - if (fsr & ARM_SMMU_FSR_SS) 327 327 + if (fsr & ARM_SMMU_CB_FSR_SS) 328 328 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 329 329 ARM_SMMU_RESUME_TERMINATE); 330 330 } ··· 383 383 struct arm_smmu_domain *smmu_domain = dev; 384 384 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 385 385 struct arm_smmu_device *smmu = smmu_domain->smmu; 386 386 - u32 fsr, fsynr, cbfrsynra, resume = 0; 386 386 + struct arm_smmu_context_fault_info cfi; 387 387 + u32 resume = 0; 387 388 int idx = smmu_domain->cfg.cbndx; 388 389 phys_addr_t phys_soft; 389 389 - unsigned long iova; 390 390 int ret, tmp; 391 391 392 392 static DEFINE_RATELIMIT_STATE(_rs, 393 393 DEFAULT_RATELIMIT_INTERVAL, 394 394 DEFAULT_RATELIMIT_BURST); 395 395 396 396 - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 397 397 - if (!(fsr & ARM_SMMU_FSR_FAULT)) 396 396 + arm_smmu_read_context_fault_info(smmu, idx, &cfi); 397 397 + 398 398 + if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) 398 399 return IRQ_NONE; 399 400 400 400 - fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 401 401 - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 402 402 - cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 403 403 - 404 401 if (list_empty(&tbu_list)) { 405 405 - ret = report_iommu_fault(&smmu_domain->domain, NULL, iova, 406 406 - fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 402 402 + ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 403 403 + cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 407 404 408 405 if (ret == -ENOSYS) 409 409 - dev_err_ratelimited(smmu->dev, 410 410 - "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 411 411 - fsr, iova, fsynr, cbfrsynra, idx); 406 406 + arm_smmu_print_context_fault_info(smmu, idx, &cfi); 412 407 413 413 - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 408 408 + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 414 409 return IRQ_HANDLED; 415 410 } 416 411 417 417 - phys_soft = ops->iova_to_phys(ops, iova); 412 412 + phys_soft = ops->iova_to_phys(ops, cfi.iova); 418 413 419 419 - tmp = report_iommu_fault(&smmu_domain->domain, NULL, iova, 420 420 - fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 414 414 + tmp = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 415 415 + cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 421 416 if (!tmp || tmp == -EBUSY) { 422 422 - dev_dbg(smmu->dev, 423 423 - "Context fault handled by client: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n", 424 424 - iova, fsr, fsynr, idx); 425 425 - dev_dbg(smmu->dev, "soft iova-to-phys=%pa\n", &phys_soft); 426 417 ret = IRQ_HANDLED; 427 418 resume = ARM_SMMU_RESUME_TERMINATE; 428 419 } else { 429 429 - phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, iova, fsr); 420 420 + phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, cfi.iova, cfi.fsr); 430 421 431 422 if (__ratelimit(&_rs)) { 432 432 - dev_err(smmu->dev, 433 433 - "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 434 434 - fsr, iova, fsynr, cbfrsynra, idx); 435 435 - dev_err(smmu->dev, 436 436 - "FSR = %08x [%s%s%s%s%s%s%s%s%s], SID=0x%x\n", 437 437 - fsr, 438 438 - (fsr & 0x02) ? "TF " : "", 439 439 - (fsr & 0x04) ? "AFF " : "", 440 440 - (fsr & 0x08) ? "PF " : "", 441 441 - (fsr & 0x10) ? "EF " : "", 442 442 - (fsr & 0x20) ? "TLBMCF " : "", 443 443 - (fsr & 0x40) ? "TLBLKF " : "", 444 444 - (fsr & 0x80) ? "MHF " : "", 445 445 - (fsr & 0x40000000) ? "SS " : "", 446 446 - (fsr & 0x80000000) ? "MULTI " : "", 447 447 - cbfrsynra); 423 423 + arm_smmu_print_context_fault_info(smmu, idx, &cfi); 448 424 449 425 dev_err(smmu->dev, 450 426 "soft iova-to-phys=%pa\n", &phys_soft); ··· 454 478 */ 455 479 if (tmp != -EBUSY) { 456 480 /* Clear the faulting FSR */ 457 457 - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 481 481 + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 458 482 459 483 /* Retry or terminate any stalled transactions */ 460 460 - if (fsr & ARM_SMMU_FSR_SS) 484 484 + if (cfi.fsr & ARM_SMMU_CB_FSR_SS) 461 485 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, resume); 462 486 } 463 487 464 488 return ret; 465 489 } 466 490 467 467 - static int qcom_tbu_probe(struct platform_device *pdev) 491 491 + int qcom_tbu_probe(struct platform_device *pdev) 468 492 { 469 493 struct of_phandle_args args = { .args_count = 2 }; 470 494 struct device_node *np = pdev->dev.of_node; ··· 506 530 507 531 return 0; 508 532 } 509 509 - 510 510 - static const struct of_device_id qcom_tbu_of_match[] = { 511 511 - { .compatible = "qcom,sc7280-tbu" }, 512 512 - { .compatible = "qcom,sdm845-tbu" }, 513 513 - { } 514 514 - }; 515 515 - 516 516 - static struct platform_driver qcom_tbu_driver = { 517 517 - .driver = { 518 518 - .name = "qcom_tbu", 519 519 - .of_match_table = qcom_tbu_of_match, 520 520 - }, 521 521 - .probe = qcom_tbu_probe, 522 522 - }; 523 523 - builtin_platform_driver(qcom_tbu_driver);

+41 -1

drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c

reviewed

··· 8 8 #include <linux/delay.h> 9 9 #include <linux/of_device.h> 10 10 #include <linux/firmware/qcom/qcom_scm.h> 11 11 + #include <linux/platform_device.h> 12 12 + #include <linux/pm_runtime.h> 11 13 12 14 #include "arm-smmu.h" 13 15 #include "arm-smmu-qcom.h" ··· 471 469 472 470 /* Check to make sure qcom_scm has finished probing */ 473 471 if (!qcom_scm_is_available()) 474 474 - return ERR_PTR(-EPROBE_DEFER); 472 472 + return ERR_PTR(dev_err_probe(smmu->dev, -EPROBE_DEFER, 473 473 + "qcom_scm not ready\n")); 475 474 476 475 qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL); 477 476 if (!qsmmu) ··· 564 561 }; 565 562 #endif 566 563 564 564 + static int qcom_smmu_tbu_probe(struct platform_device *pdev) 565 565 + { 566 566 + struct device *dev = &pdev->dev; 567 567 + int ret; 568 568 + 569 569 + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM_DEBUG)) { 570 570 + ret = qcom_tbu_probe(pdev); 571 571 + if (ret) 572 572 + return ret; 573 573 + } 574 574 + 575 575 + if (dev->pm_domain) { 576 576 + pm_runtime_set_active(dev); 577 577 + pm_runtime_enable(dev); 578 578 + } 579 579 + 580 580 + return 0; 581 581 + } 582 582 + 583 583 + static const struct of_device_id qcom_smmu_tbu_of_match[] = { 584 584 + { .compatible = "qcom,sc7280-tbu" }, 585 585 + { .compatible = "qcom,sdm845-tbu" }, 586 586 + { } 587 587 + }; 588 588 + 589 589 + static struct platform_driver qcom_smmu_tbu_driver = { 590 590 + .driver = { 591 591 + .name = "qcom_tbu", 592 592 + .of_match_table = qcom_smmu_tbu_of_match, 593 593 + }, 594 594 + .probe = qcom_smmu_tbu_probe, 595 595 + }; 596 596 + 567 597 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) 568 598 { 569 599 const struct device_node *np = smmu->dev->of_node; 570 600 const struct of_device_id *match; 601 601 + static u8 tbu_registered; 602 602 + 603 603 + if (!tbu_registered++) 604 604 + platform_driver_register(&qcom_smmu_tbu_driver); 571 605 572 606 #ifdef CONFIG_ACPI 573 607 if (np == NULL) {

+2

drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h

reviewed

··· 34 34 35 35 #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG 36 36 void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu); 37 37 + int qcom_tbu_probe(struct platform_device *pdev); 37 38 #else 38 39 static inline void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) { } 40 40 + static inline int qcom_tbu_probe(struct platform_device *pdev) { return -EINVAL; } 39 41 #endif 40 42 41 43 #endif /* _ARM_SMMU_QCOM_H */

+58 -19

drivers/iommu/arm/arm-smmu/arm-smmu.c

reviewed

··· 178 178 it.cur_count = 1; 179 179 } 180 180 181 181 - err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, 182 182 - &arm_smmu_ops); 181 181 + err = iommu_fwspec_init(dev, NULL); 183 182 if (err) 184 183 return err; 185 184 ··· 404 405 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 405 406 }; 406 407 408 408 + 409 409 + void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx, 410 410 + struct arm_smmu_context_fault_info *cfi) 411 411 + { 412 412 + cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 413 413 + cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 414 414 + cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 415 415 + cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 416 416 + } 417 417 + 418 418 + void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx, 419 419 + const struct arm_smmu_context_fault_info *cfi) 420 420 + { 421 421 + dev_dbg(smmu->dev, 422 422 + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 423 423 + cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx); 424 424 + 425 425 + dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n", 426 426 + cfi->fsr, 427 427 + (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "", 428 428 + (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "", 429 429 + (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr), 430 430 + (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "", 431 431 + (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "", 432 432 + (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "", 433 433 + (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "", 434 434 + (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "", 435 435 + (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "", 436 436 + (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "", 437 437 + (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "", 438 438 + cfi->cbfrsynra); 439 439 + 440 440 + dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n", 441 441 + cfi->fsynr, 442 442 + (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr), 443 443 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "", 444 444 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "", 445 445 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "", 446 446 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "", 447 447 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "", 448 448 + (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "", 449 449 + (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr)); 450 450 + } 451 451 + 407 452 static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 408 453 { 409 409 - u32 fsr, fsynr, cbfrsynra; 410 410 - unsigned long iova; 454 454 + struct arm_smmu_context_fault_info cfi; 411 455 struct arm_smmu_domain *smmu_domain = dev; 412 456 struct arm_smmu_device *smmu = smmu_domain->smmu; 457 457 + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 458 458 + DEFAULT_RATELIMIT_BURST); 413 459 int idx = smmu_domain->cfg.cbndx; 414 460 int ret; 415 461 416 416 - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 417 417 - if (!(fsr & ARM_SMMU_FSR_FAULT)) 462 462 + arm_smmu_read_context_fault_info(smmu, idx, &cfi); 463 463 + 464 464 + if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) 418 465 return IRQ_NONE; 419 466 420 420 - fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 421 421 - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 422 422 - cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 467 467 + ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 468 468 + cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 423 469 424 424 - ret = report_iommu_fault(&smmu_domain->domain, NULL, iova, 425 425 - fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 470 470 + if (ret == -ENOSYS && __ratelimit(&rs)) 471 471 + arm_smmu_print_context_fault_info(smmu, idx, &cfi); 426 472 427 427 - if (ret == -ENOSYS) 428 428 - dev_err_ratelimited(smmu->dev, 429 429 - "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 430 430 - fsr, iova, fsynr, cbfrsynra, idx); 431 431 - 432 432 - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 473 473 + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 433 474 return IRQ_HANDLED; 434 475 } 435 476 ··· 1345 1306 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1346 1307 1347 1308 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1348 1348 - if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE), 1309 1309 + if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE), 1349 1310 5, 50)) { 1350 1311 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1351 1312 dev_err(dev, ··· 1681 1642 /* Make sure all context banks are disabled and clear CB_FSR */ 1682 1643 for (i = 0; i < smmu->num_context_banks; ++i) { 1683 1644 arm_smmu_write_context_bank(smmu, i); 1684 1684 - arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); 1645 1645 + arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT); 1685 1646 } 1686 1647 1687 1648 /* Invalidate the TLB, just in case */

+44 -23

drivers/iommu/arm/arm-smmu/arm-smmu.h

reviewed

··· 196 196 #define ARM_SMMU_CB_PAR_F BIT(0) 197 197 198 198 #define ARM_SMMU_CB_FSR 0x58 199 199 - #define ARM_SMMU_FSR_MULTI BIT(31) 200 200 - #define ARM_SMMU_FSR_SS BIT(30) 201 201 - #define ARM_SMMU_FSR_UUT BIT(8) 202 202 - #define ARM_SMMU_FSR_ASF BIT(7) 203 203 - #define ARM_SMMU_FSR_TLBLKF BIT(6) 204 204 - #define ARM_SMMU_FSR_TLBMCF BIT(5) 205 205 - #define ARM_SMMU_FSR_EF BIT(4) 206 206 - #define ARM_SMMU_FSR_PF BIT(3) 207 207 - #define ARM_SMMU_FSR_AFF BIT(2) 208 208 - #define ARM_SMMU_FSR_TF BIT(1) 199 199 + #define ARM_SMMU_CB_FSR_MULTI BIT(31) 200 200 + #define ARM_SMMU_CB_FSR_SS BIT(30) 201 201 + #define ARM_SMMU_CB_FSR_FORMAT GENMASK(10, 9) 202 202 + #define ARM_SMMU_CB_FSR_UUT BIT(8) 203 203 + #define ARM_SMMU_CB_FSR_ASF BIT(7) 204 204 + #define ARM_SMMU_CB_FSR_TLBLKF BIT(6) 205 205 + #define ARM_SMMU_CB_FSR_TLBMCF BIT(5) 206 206 + #define ARM_SMMU_CB_FSR_EF BIT(4) 207 207 + #define ARM_SMMU_CB_FSR_PF BIT(3) 208 208 + #define ARM_SMMU_CB_FSR_AFF BIT(2) 209 209 + #define ARM_SMMU_CB_FSR_TF BIT(1) 209 210 210 210 - #define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF | \ 211 211 - ARM_SMMU_FSR_ASF | \ 212 212 - ARM_SMMU_FSR_TLBMCF | \ 213 213 - ARM_SMMU_FSR_TLBLKF) 211 211 + #define ARM_SMMU_CB_FSR_IGN (ARM_SMMU_CB_FSR_AFF | \ 212 212 + ARM_SMMU_CB_FSR_ASF | \ 213 213 + ARM_SMMU_CB_FSR_TLBMCF | \ 214 214 + ARM_SMMU_CB_FSR_TLBLKF) 214 215 215 215 - #define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI | \ 216 216 - ARM_SMMU_FSR_SS | \ 217 217 - ARM_SMMU_FSR_UUT | \ 218 218 - ARM_SMMU_FSR_EF | \ 219 219 - ARM_SMMU_FSR_PF | \ 220 220 - ARM_SMMU_FSR_TF | \ 221 221 - ARM_SMMU_FSR_IGN) 216 216 + #define ARM_SMMU_CB_FSR_FAULT (ARM_SMMU_CB_FSR_MULTI | \ 217 217 + ARM_SMMU_CB_FSR_SS | \ 218 218 + ARM_SMMU_CB_FSR_UUT | \ 219 219 + ARM_SMMU_CB_FSR_EF | \ 220 220 + ARM_SMMU_CB_FSR_PF | \ 221 221 + ARM_SMMU_CB_FSR_TF | \ 222 222 + ARM_SMMU_CB_FSR_IGN) 222 223 223 224 #define ARM_SMMU_CB_FAR 0x60 224 225 225 226 #define ARM_SMMU_CB_FSYNR0 0x68 226 226 - #define ARM_SMMU_FSYNR0_WNR BIT(4) 227 227 + #define ARM_SMMU_CB_FSYNR0_PLVL GENMASK(1, 0) 228 228 + #define ARM_SMMU_CB_FSYNR0_WNR BIT(4) 229 229 + #define ARM_SMMU_CB_FSYNR0_PNU BIT(5) 230 230 + #define ARM_SMMU_CB_FSYNR0_IND BIT(6) 231 231 + #define ARM_SMMU_CB_FSYNR0_NSATTR BIT(8) 232 232 + #define ARM_SMMU_CB_FSYNR0_PTWF BIT(10) 233 233 + #define ARM_SMMU_CB_FSYNR0_AFR BIT(11) 234 234 + #define ARM_SMMU_CB_FSYNR0_S1CBNDX GENMASK(23, 16) 227 235 228 236 #define ARM_SMMU_CB_FSYNR1 0x6c 229 237 ··· 245 237 #define ARM_SMMU_CB_ATS1PR 0x800 246 238 247 239 #define ARM_SMMU_CB_ATSR 0x8f0 248 248 - #define ARM_SMMU_ATSR_ACTIVE BIT(0) 240 240 + #define ARM_SMMU_CB_ATSR_ACTIVE BIT(0) 249 241 250 242 #define ARM_SMMU_RESUME_TERMINATE BIT(0) 251 243 ··· 540 532 541 533 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); 542 534 int arm_mmu500_reset(struct arm_smmu_device *smmu); 535 535 + 536 536 + struct arm_smmu_context_fault_info { 537 537 + unsigned long iova; 538 538 + u32 fsr; 539 539 + u32 fsynr; 540 540 + u32 cbfrsynra; 541 541 + }; 542 542 + 543 543 + void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx, 544 544 + struct arm_smmu_context_fault_info *cfi); 545 545 + 546 546 + void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx, 547 547 + const struct arm_smmu_context_fault_info *cfi); 543 548 544 549 #endif /* _ARM_SMMU_H */

+2 -2

drivers/iommu/arm/arm-smmu/qcom_iommu.c

reviewed

··· 194 194 195 195 fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR); 196 196 197 197 - if (!(fsr & ARM_SMMU_FSR_FAULT)) 197 197 + if (!(fsr & ARM_SMMU_CB_FSR_FAULT)) 198 198 return IRQ_NONE; 199 199 200 200 fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); ··· 274 274 275 275 /* Clear context bank fault address fault status registers */ 276 276 iommu_writel(ctx, ARM_SMMU_CB_FAR, 0); 277 277 - iommu_writel(ctx, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); 277 277 + iommu_writel(ctx, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT); 278 278 279 279 /* TTBRs */ 280 280 iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,

+6 -10

drivers/iommu/dma-iommu.c

reviewed

··· 939 939 * but an IOMMU which supports smaller pages might not map the whole thing. 940 940 */ 941 941 static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, 942 942 - size_t size, struct sg_table *sgt, gfp_t gfp, pgprot_t prot, 943 943 - unsigned long attrs) 942 942 + size_t size, struct sg_table *sgt, gfp_t gfp, unsigned long attrs) 944 943 { 945 944 struct iommu_domain *domain = iommu_get_dma_domain(dev); 946 945 struct iommu_dma_cookie *cookie = domain->iova_cookie; ··· 1013 1014 } 1014 1015 1015 1016 static void *iommu_dma_alloc_remap(struct device *dev, size_t size, 1016 1016 - dma_addr_t *dma_handle, gfp_t gfp, pgprot_t prot, 1017 1017 - unsigned long attrs) 1017 1017 + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 1018 1018 { 1019 1019 struct page **pages; 1020 1020 struct sg_table sgt; 1021 1021 void *vaddr; 1022 1022 + pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); 1022 1023 1023 1023 - pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, prot, 1024 1024 - attrs); 1024 1024 + pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, attrs); 1025 1025 if (!pages) 1026 1026 return NULL; 1027 1027 *dma_handle = sgt.sgl->dma_address; ··· 1047 1049 if (!sh) 1048 1050 return NULL; 1049 1051 1050 1050 - sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp, 1051 1051 - PAGE_KERNEL, attrs); 1052 1052 + sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp, attrs); 1052 1053 if (!sh->pages) { 1053 1054 kfree(sh); 1054 1055 return NULL; ··· 1616 1619 1617 1620 if (gfpflags_allow_blocking(gfp) && 1618 1621 !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) { 1619 1619 - return iommu_dma_alloc_remap(dev, size, handle, gfp, 1620 1620 - dma_pgprot(dev, PAGE_KERNEL, attrs), attrs); 1622 1622 + return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs); 1621 1623 } 1622 1624 1623 1625 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&

+2 -1

drivers/iommu/intel/cache.c

reviewed

··· 245 245 * shared_bits are all equal in both pfn and end_pfn. 246 246 */ 247 247 shared_bits = ~(pfn ^ end_pfn) & ~bitmask; 248 248 - mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG; 248 248 + mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH; 249 249 + aligned_pages = 1UL << mask; 249 250 } 250 251 251 252 *_pages = aligned_pages;

+1 -1

drivers/iommu/intel/dmar.c

reviewed

··· 1446 1446 */ 1447 1447 writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG); 1448 1448 1449 1449 - while (qi->desc_status[wait_index] != QI_DONE) { 1449 1449 + while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) { 1450 1450 /* 1451 1451 * We will leave the interrupts disabled, to prevent interrupt 1452 1452 * context to queue another cmd while a cmd is already submitted

+137 -59

drivers/iommu/intel/iommu.c

reviewed

··· 854 854 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE); 855 855 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE; 856 856 if (domain->use_first_level) 857 857 - pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 857 857 + pteval |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 858 858 859 859 tmp = 0ULL; 860 860 if (!try_cmpxchg64(&pte->val, &tmp, pteval)) ··· 1359 1359 } 1360 1360 } 1361 1361 1362 1362 - static void __iommu_flush_dev_iotlb(struct device_domain_info *info, 1363 1363 - u64 addr, unsigned int mask) 1364 1364 - { 1365 1365 - u16 sid, qdep; 1366 1366 - 1367 1367 - if (!info || !info->ats_enabled) 1368 1368 - return; 1369 1369 - 1370 1370 - sid = info->bus << 8 | info->devfn; 1371 1371 - qdep = info->ats_qdep; 1372 1372 - qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, 1373 1373 - qdep, addr, mask); 1374 1374 - quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep); 1375 1375 - } 1376 1376 - 1377 1362 static void intel_flush_iotlb_all(struct iommu_domain *domain) 1378 1363 { 1379 1364 cache_tag_flush_all(to_dmar_domain(domain)); ··· 1857 1872 attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); 1858 1873 attr |= DMA_FL_PTE_PRESENT; 1859 1874 if (domain->use_first_level) { 1860 1860 - attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 1875 1875 + attr |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 1861 1876 if (prot & DMA_PTE_WRITE) 1862 1877 attr |= DMA_FL_PTE_DIRTY; 1863 1878 } ··· 1944 1959 { 1945 1960 struct intel_iommu *iommu = info->iommu; 1946 1961 struct context_entry *context; 1947 1947 - u16 did_old; 1948 1962 1949 1963 spin_lock(&iommu->lock); 1950 1964 context = iommu_context_addr(iommu, bus, devfn, 0); ··· 1952 1968 return; 1953 1969 } 1954 1970 1955 1955 - did_old = context_domain_id(context); 1956 1956 - 1957 1971 context_clear_entry(context); 1958 1972 __iommu_flush_cache(iommu, context, sizeof(*context)); 1959 1973 spin_unlock(&iommu->lock); 1960 1960 - iommu->flush.flush_context(iommu, 1961 1961 - did_old, 1962 1962 - (((u16)bus) << 8) | devfn, 1963 1963 - DMA_CCMD_MASK_NOBIT, 1964 1964 - DMA_CCMD_DEVICE_INVL); 1965 1965 - 1966 1966 - iommu->flush.flush_iotlb(iommu, 1967 1967 - did_old, 1968 1968 - 0, 1969 1969 - 0, 1970 1970 - DMA_TLB_DSI_FLUSH); 1971 1971 - 1972 1972 - __iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH); 1974 1974 + intel_context_flush_present(info, context, true); 1973 1975 } 1974 1976 1975 1977 static int domain_setup_first_level(struct intel_iommu *iommu, ··· 2041 2071 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 2042 2072 ret = iommu_domain_identity_map(si_domain, 2043 2073 mm_to_dma_pfn_start(start_pfn), 2044 2044 - mm_to_dma_pfn_end(end_pfn)); 2074 2074 + mm_to_dma_pfn_end(end_pfn-1)); 2045 2075 if (ret) 2046 2076 return ret; 2047 2077 } ··· 2147 2177 return false; 2148 2178 } 2149 2179 2150 2150 - /* 2151 2151 - * Return the required default domain type for a specific device. 2152 2152 - * 2153 2153 - * @dev: the device in query 2154 2154 - * @startup: true if this is during early boot 2155 2155 - * 2156 2156 - * Returns: 2157 2157 - * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain 2158 2158 - * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain 2159 2159 - * - 0: both identity and dynamic domains work for this device 2160 2160 - */ 2161 2180 static int device_def_domain_type(struct device *dev) 2162 2181 { 2163 2182 if (dev_is_pci(dev)) { ··· 3592 3633 } 3593 3634 }; 3594 3635 3636 3636 + static int iommu_superpage_capability(struct intel_iommu *iommu, bool first_stage) 3637 3637 + { 3638 3638 + if (!intel_iommu_superpage) 3639 3639 + return 0; 3640 3640 + 3641 3641 + if (first_stage) 3642 3642 + return cap_fl1gp_support(iommu->cap) ? 2 : 1; 3643 3643 + 3644 3644 + return fls(cap_super_page_val(iommu->cap)); 3645 3645 + } 3646 3646 + 3647 3647 + static struct dmar_domain *paging_domain_alloc(struct device *dev, bool first_stage) 3648 3648 + { 3649 3649 + struct device_domain_info *info = dev_iommu_priv_get(dev); 3650 3650 + struct intel_iommu *iommu = info->iommu; 3651 3651 + struct dmar_domain *domain; 3652 3652 + int addr_width; 3653 3653 + 3654 3654 + domain = kzalloc(sizeof(*domain), GFP_KERNEL); 3655 3655 + if (!domain) 3656 3656 + return ERR_PTR(-ENOMEM); 3657 3657 + 3658 3658 + INIT_LIST_HEAD(&domain->devices); 3659 3659 + INIT_LIST_HEAD(&domain->dev_pasids); 3660 3660 + INIT_LIST_HEAD(&domain->cache_tags); 3661 3661 + spin_lock_init(&domain->lock); 3662 3662 + spin_lock_init(&domain->cache_lock); 3663 3663 + xa_init(&domain->iommu_array); 3664 3664 + 3665 3665 + domain->nid = dev_to_node(dev); 3666 3666 + domain->has_iotlb_device = info->ats_enabled; 3667 3667 + domain->use_first_level = first_stage; 3668 3668 + 3669 3669 + /* calculate the address width */ 3670 3670 + addr_width = agaw_to_width(iommu->agaw); 3671 3671 + if (addr_width > cap_mgaw(iommu->cap)) 3672 3672 + addr_width = cap_mgaw(iommu->cap); 3673 3673 + domain->gaw = addr_width; 3674 3674 + domain->agaw = iommu->agaw; 3675 3675 + domain->max_addr = __DOMAIN_MAX_ADDR(addr_width); 3676 3676 + 3677 3677 + /* iommu memory access coherency */ 3678 3678 + domain->iommu_coherency = iommu_paging_structure_coherency(iommu); 3679 3679 + 3680 3680 + /* pagesize bitmap */ 3681 3681 + domain->domain.pgsize_bitmap = SZ_4K; 3682 3682 + domain->iommu_superpage = iommu_superpage_capability(iommu, first_stage); 3683 3683 + domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain); 3684 3684 + 3685 3685 + /* 3686 3686 + * IOVA aperture: First-level translation restricts the input-address 3687 3687 + * to a canonical address (i.e., address bits 63:N have the same value 3688 3688 + * as address bit [N-1], where N is 48-bits with 4-level paging and 3689 3689 + * 57-bits with 5-level paging). Hence, skip bit [N-1]. 3690 3690 + */ 3691 3691 + domain->domain.geometry.force_aperture = true; 3692 3692 + domain->domain.geometry.aperture_start = 0; 3693 3693 + if (first_stage) 3694 3694 + domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1); 3695 3695 + else 3696 3696 + domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw); 3697 3697 + 3698 3698 + /* always allocate the top pgd */ 3699 3699 + domain->pgd = iommu_alloc_page_node(domain->nid, GFP_KERNEL); 3700 3700 + if (!domain->pgd) { 3701 3701 + kfree(domain); 3702 3702 + return ERR_PTR(-ENOMEM); 3703 3703 + } 3704 3704 + domain_flush_cache(domain, domain->pgd, PAGE_SIZE); 3705 3705 + 3706 3706 + return domain; 3707 3707 + } 3708 3708 + 3595 3709 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) 3596 3710 { 3597 3711 struct dmar_domain *dmar_domain; ··· 3727 3695 if (user_data || (dirty_tracking && !ssads_supported(iommu))) 3728 3696 return ERR_PTR(-EOPNOTSUPP); 3729 3697 3730 3730 - /* 3731 3731 - * domain_alloc_user op needs to fully initialize a domain before 3732 3732 - * return, so uses iommu_domain_alloc() here for simple. 3733 3733 - */ 3734 3734 - domain = iommu_domain_alloc(dev->bus); 3735 3735 - if (!domain) 3736 3736 - return ERR_PTR(-ENOMEM); 3737 3737 - 3738 3738 - dmar_domain = to_dmar_domain(domain); 3698 3698 + /* Do not use first stage for user domain translation. */ 3699 3699 + dmar_domain = paging_domain_alloc(dev, false); 3700 3700 + if (IS_ERR(dmar_domain)) 3701 3701 + return ERR_CAST(dmar_domain); 3702 3702 + domain = &dmar_domain->domain; 3703 3703 + domain->type = IOMMU_DOMAIN_UNMANAGED; 3704 3704 + domain->owner = &intel_iommu_ops; 3705 3705 + domain->ops = intel_iommu_ops.default_domain_ops; 3739 3706 3740 3707 if (nested_parent) { 3741 3708 dmar_domain->nested_parent = true; ··· 4244 4213 return 0; 4245 4214 } 4246 4215 4216 4216 + static int context_flip_pri(struct device_domain_info *info, bool enable) 4217 4217 + { 4218 4218 + struct intel_iommu *iommu = info->iommu; 4219 4219 + u8 bus = info->bus, devfn = info->devfn; 4220 4220 + struct context_entry *context; 4221 4221 + 4222 4222 + spin_lock(&iommu->lock); 4223 4223 + if (context_copied(iommu, bus, devfn)) { 4224 4224 + spin_unlock(&iommu->lock); 4225 4225 + return -EINVAL; 4226 4226 + } 4227 4227 + 4228 4228 + context = iommu_context_addr(iommu, bus, devfn, false); 4229 4229 + if (!context || !context_present(context)) { 4230 4230 + spin_unlock(&iommu->lock); 4231 4231 + return -ENODEV; 4232 4232 + } 4233 4233 + 4234 4234 + if (enable) 4235 4235 + context_set_sm_pre(context); 4236 4236 + else 4237 4237 + context_clear_sm_pre(context); 4238 4238 + 4239 4239 + if (!ecap_coherent(iommu->ecap)) 4240 4240 + clflush_cache_range(context, sizeof(*context)); 4241 4241 + intel_context_flush_present(info, context, true); 4242 4242 + spin_unlock(&iommu->lock); 4243 4243 + 4244 4244 + return 0; 4245 4245 + } 4246 4246 + 4247 4247 static int intel_iommu_enable_iopf(struct device *dev) 4248 4248 { 4249 4249 struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL; ··· 4304 4242 if (ret) 4305 4243 return ret; 4306 4244 4245 4245 + ret = context_flip_pri(info, true); 4246 4246 + if (ret) 4247 4247 + goto err_remove_device; 4248 4248 + 4307 4249 ret = pci_enable_pri(pdev, PRQ_DEPTH); 4308 4308 - if (ret) { 4309 4309 - iopf_queue_remove_device(iommu->iopf_queue, dev); 4310 4310 - return ret; 4311 4311 - } 4250 4250 + if (ret) 4251 4251 + goto err_clear_pri; 4312 4252 4313 4253 info->pri_enabled = 1; 4314 4254 4315 4255 return 0; 4256 4256 + err_clear_pri: 4257 4257 + context_flip_pri(info, false); 4258 4258 + err_remove_device: 4259 4259 + iopf_queue_remove_device(iommu->iopf_queue, dev); 4260 4260 + 4261 4261 + return ret; 4316 4262 } 4317 4263 4318 4264 static int intel_iommu_disable_iopf(struct device *dev) ··· 4330 4260 4331 4261 if (!info->pri_enabled) 4332 4262 return -EINVAL; 4263 4263 + 4264 4264 + /* Disable new PRI reception: */ 4265 4265 + context_flip_pri(info, false); 4266 4266 + 4267 4267 + /* 4268 4268 + * Remove device from fault queue and acknowledge all outstanding 4269 4269 + * PRQs to the device: 4270 4270 + */ 4271 4271 + iopf_queue_remove_device(iommu->iopf_queue, dev); 4333 4272 4334 4273 /* 4335 4274 * PCIe spec states that by clearing PRI enable bit, the Page ··· 4350 4271 */ 4351 4272 pci_disable_pri(to_pci_dev(dev)); 4352 4273 info->pri_enabled = 0; 4353 4353 - iopf_queue_remove_device(iommu->iopf_queue, dev); 4354 4274 4355 4275 return 0; 4356 4276 }

+15 -4

drivers/iommu/intel/iommu.h

reviewed

··· 49 49 #define DMA_FL_PTE_US BIT_ULL(2) 50 50 #define DMA_FL_PTE_ACCESS BIT_ULL(5) 51 51 #define DMA_FL_PTE_DIRTY BIT_ULL(6) 52 52 - #define DMA_FL_PTE_XD BIT_ULL(63) 53 52 54 53 #define DMA_SL_PTE_DIRTY_BIT 9 55 54 #define DMA_SL_PTE_DIRTY BIT_ULL(DMA_SL_PTE_DIRTY_BIT) ··· 830 831 static inline u64 dma_pte_addr(struct dma_pte *pte) 831 832 { 832 833 #ifdef CONFIG_64BIT 833 833 - return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD); 834 834 + return pte->val & VTD_PAGE_MASK; 834 835 #else 835 836 /* Must have a full atomic 64-bit read */ 836 836 - return __cmpxchg64(&pte->val, 0ULL, 0ULL) & 837 837 - VTD_PAGE_MASK & (~DMA_FL_PTE_XD); 837 837 + return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK; 838 838 #endif 839 839 } 840 840 ··· 1045 1047 context->lo |= BIT_ULL(4); 1046 1048 } 1047 1049 1050 1050 + /* 1051 1051 + * Clear the PRE(Page Request Enable) field of a scalable mode context 1052 1052 + * entry. 1053 1053 + */ 1054 1054 + static inline void context_clear_sm_pre(struct context_entry *context) 1055 1055 + { 1056 1056 + context->lo &= ~BIT_ULL(4); 1057 1057 + } 1058 1058 + 1048 1059 /* Returns a number of VTD pages, but aligned to MM page size */ 1049 1060 static inline unsigned long aligned_nrpages(unsigned long host_addr, size_t size) 1050 1061 { ··· 1151 1144 void cache_tag_flush_all(struct dmar_domain *domain); 1152 1145 void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, 1153 1146 unsigned long end); 1147 1147 + 1148 1148 + void intel_context_flush_present(struct device_domain_info *info, 1149 1149 + struct context_entry *context, 1150 1150 + bool affect_domains); 1154 1151 1155 1152 #ifdef CONFIG_INTEL_IOMMU_SVM 1156 1153 void intel_svm_check(struct intel_iommu *iommu);

+2 -2

drivers/iommu/intel/irq_remapping.c

reviewed

··· 597 597 598 598 if (ir_pre_enabled(iommu)) { 599 599 if (!is_kdump_kernel()) { 600 600 - pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", 601 601 - iommu->name); 600 600 + pr_info_once("IRQ remapping was enabled on %s but we are not in kdump mode\n", 601 601 + iommu->name); 602 602 clear_ir_pre_enabled(iommu); 603 603 iommu_disable_irq_remapping(iommu); 604 604 } else if (iommu_load_old_irte(iommu))

+92 -24

drivers/iommu/intel/pasid.c

reviewed

··· 146 146 retry: 147 147 entries = get_pasid_table_from_pde(&dir[dir_index]); 148 148 if (!entries) { 149 149 + u64 tmp; 150 150 + 149 151 entries = iommu_alloc_page_node(info->iommu->node, GFP_ATOMIC); 150 152 if (!entries) 151 153 return NULL; ··· 158 156 * clear. However, this entry might be populated by others 159 157 * while we are preparing it. Use theirs with a retry. 160 158 */ 161 161 - if (cmpxchg64(&dir[dir_index].val, 0ULL, 162 162 - (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { 159 159 + tmp = 0ULL; 160 160 + if (!try_cmpxchg64(&dir[dir_index].val, &tmp, 161 161 + (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { 163 162 iommu_free_page(entries); 164 163 goto retry; 165 164 } ··· 336 333 pasid_set_domain_id(pte, did); 337 334 pasid_set_address_width(pte, iommu->agaw); 338 335 pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); 339 339 - pasid_set_nxe(pte); 340 336 341 337 /* Setup Present and PASID Granular Transfer Type: */ 342 338 pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY); ··· 694 692 context_clear_entry(context); 695 693 __iommu_flush_cache(iommu, context, sizeof(*context)); 696 694 spin_unlock(&iommu->lock); 697 697 - 698 698 - /* 699 699 - * Cache invalidation for changes to a scalable-mode context table 700 700 - * entry. 701 701 - * 702 702 - * Section 6.5.3.3 of the VT-d spec: 703 703 - * - Device-selective context-cache invalidation; 704 704 - * - Domain-selective PASID-cache invalidation to affected domains 705 705 - * (can be skipped if all PASID entries were not-present); 706 706 - * - Domain-selective IOTLB invalidation to affected domains; 707 707 - * - Global Device-TLB invalidation to affected functions. 708 708 - * 709 709 - * The iommu has been parked in the blocking state. All domains have 710 710 - * been detached from the device or PASID. The PASID and IOTLB caches 711 711 - * have been invalidated during the domain detach path. 712 712 - */ 713 713 - iommu->flush.flush_context(iommu, 0, PCI_DEVID(bus, devfn), 714 714 - DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); 715 715 - devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); 695 695 + intel_context_flush_present(info, context, false); 716 696 } 717 697 718 698 static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) ··· 752 768 753 769 if (info->ats_supported) 754 770 context_set_sm_dte(context); 755 755 - if (info->pri_supported) 756 756 - context_set_sm_pre(context); 757 771 if (info->pasid_supported) 758 772 context_set_pasid(context); 759 773 ··· 853 871 return device_pasid_table_setup(dev, info->bus, info->devfn); 854 872 855 873 return pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_setup, dev); 874 874 + } 875 875 + 876 876 + /* 877 877 + * Global Device-TLB invalidation following changes in a context entry which 878 878 + * was present. 879 879 + */ 880 880 + static void __context_flush_dev_iotlb(struct device_domain_info *info) 881 881 + { 882 882 + if (!info->ats_enabled) 883 883 + return; 884 884 + 885 885 + qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn), 886 886 + info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH); 887 887 + 888 888 + /* 889 889 + * There is no guarantee that the device DMA is stopped when it reaches 890 890 + * here. Therefore, always attempt the extra device TLB invalidation 891 891 + * quirk. The impact on performance is acceptable since this is not a 892 892 + * performance-critical path. 893 893 + */ 894 894 + quirk_extra_dev_tlb_flush(info, 0, MAX_AGAW_PFN_WIDTH, IOMMU_NO_PASID, 895 895 + info->ats_qdep); 896 896 + } 897 897 + 898 898 + /* 899 899 + * Cache invalidations after change in a context table entry that was present 900 900 + * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If 901 901 + * IOMMU is in scalable mode and all PASID table entries of the device were 902 902 + * non-present, set flush_domains to false. Otherwise, true. 903 903 + */ 904 904 + void intel_context_flush_present(struct device_domain_info *info, 905 905 + struct context_entry *context, 906 906 + bool flush_domains) 907 907 + { 908 908 + struct intel_iommu *iommu = info->iommu; 909 909 + u16 did = context_domain_id(context); 910 910 + struct pasid_entry *pte; 911 911 + int i; 912 912 + 913 913 + /* 914 914 + * Device-selective context-cache invalidation. The Domain-ID field 915 915 + * of the Context-cache Invalidate Descriptor is ignored by hardware 916 916 + * when operating in scalable mode. Therefore the @did value doesn't 917 917 + * matter in scalable mode. 918 918 + */ 919 919 + iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn), 920 920 + DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); 921 921 + 922 922 + /* 923 923 + * For legacy mode: 924 924 + * - Domain-selective IOTLB invalidation 925 925 + * - Global Device-TLB invalidation to all affected functions 926 926 + */ 927 927 + if (!sm_supported(iommu)) { 928 928 + iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); 929 929 + __context_flush_dev_iotlb(info); 930 930 + 931 931 + return; 932 932 + } 933 933 + 934 934 + /* 935 935 + * For scalable mode: 936 936 + * - Domain-selective PASID-cache invalidation to affected domains 937 937 + * - Domain-selective IOTLB invalidation to affected domains 938 938 + * - Global Device-TLB invalidation to affected functions 939 939 + */ 940 940 + if (flush_domains) { 941 941 + /* 942 942 + * If the IOMMU is running in scalable mode and there might 943 943 + * be potential PASID translations, the caller should hold 944 944 + * the lock to ensure that context changes and cache flushes 945 945 + * are atomic. 946 946 + */ 947 947 + assert_spin_locked(&iommu->lock); 948 948 + for (i = 0; i < info->pasid_table->max_pasid; i++) { 949 949 + pte = intel_pasid_get_entry(info->dev, i); 950 950 + if (!pte || !pasid_pte_is_present(pte)) 951 951 + continue; 952 952 + 953 953 + did = pasid_get_domain_id(pte); 954 954 + qi_flush_pasid_cache(iommu, did, QI_PC_ALL_PASIDS, 0); 955 955 + iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); 956 956 + } 957 957 + } 958 958 + 959 959 + __context_flush_dev_iotlb(info); 856 960 }

-10

drivers/iommu/intel/pasid.h

reviewed

··· 248 248 } 249 249 250 250 /* 251 251 - * Setup No Execute Enable bit (Bit 133) of a scalable mode PASID 252 252 - * entry. It is required when XD bit of the first level page table 253 253 - * entry is about to be set. 254 254 - */ 255 255 - static inline void pasid_set_nxe(struct pasid_entry *pe) 256 256 - { 257 257 - pasid_set_bits(&pe->val[2], 1 << 5, 1 << 5); 258 258 - } 259 259 - 260 260 - /* 261 251 * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode 262 252 * PASID entry. 263 253 */

+116 -3

drivers/iommu/io-pgtable-arm.c

reviewed

··· 76 76 77 77 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) 78 78 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) 79 79 + #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51) 79 80 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) 80 81 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) 81 82 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) ··· 86 85 87 86 #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) 88 87 /* Ignore the contiguous bit for block splitting */ 89 89 - #define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) 88 88 + #define ARM_LPAE_PTE_ATTR_HI_MASK (ARM_LPAE_PTE_XN | ARM_LPAE_PTE_DBM) 90 89 #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ 91 90 ARM_LPAE_PTE_ATTR_HI_MASK) 92 91 /* Software bit for solving coherency races */ ··· 94 93 95 94 /* Stage-1 PTE */ 96 95 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) 97 97 - #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) 96 96 + #define ARM_LPAE_PTE_AP_RDONLY_BIT 7 97 97 + #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \ 98 98 + ARM_LPAE_PTE_AP_RDONLY_BIT) 99 99 + #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \ 100 100 + ARM_LPAE_PTE_DBM) 98 101 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 99 102 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) 100 103 ··· 144 139 145 140 #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) 146 141 142 142 + #define iopte_writeable_dirty(pte) \ 143 143 + (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM) 144 144 + 145 145 + #define iopte_set_writeable_clean(ptep) \ 146 146 + set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep)) 147 147 + 147 148 struct arm_lpae_io_pgtable { 148 149 struct io_pgtable iop; 149 150 ··· 169 158 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE; 170 159 171 160 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK; 161 161 + } 162 162 + 163 163 + static inline bool iopte_table(arm_lpae_iopte pte, int lvl) 164 164 + { 165 165 + if (lvl == (ARM_LPAE_MAX_LEVELS - 1)) 166 166 + return false; 167 167 + return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE; 172 168 } 173 169 174 170 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, ··· 440 422 pte = ARM_LPAE_PTE_nG; 441 423 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 442 424 pte |= ARM_LPAE_PTE_AP_RDONLY; 425 425 + else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD) 426 426 + pte |= ARM_LPAE_PTE_DBM; 443 427 if (!(prot & IOMMU_PRIV)) 444 428 pte |= ARM_LPAE_PTE_AP_UNPRIV; 445 429 } else { ··· 746 726 return iopte_to_paddr(pte, data) | iova; 747 727 } 748 728 729 729 + struct io_pgtable_walk_data { 730 730 + struct iommu_dirty_bitmap *dirty; 731 731 + unsigned long flags; 732 732 + u64 addr; 733 733 + const u64 end; 734 734 + }; 735 735 + 736 736 + static int __arm_lpae_iopte_walk_dirty(struct arm_lpae_io_pgtable *data, 737 737 + struct io_pgtable_walk_data *walk_data, 738 738 + arm_lpae_iopte *ptep, 739 739 + int lvl); 740 740 + 741 741 + static int io_pgtable_visit_dirty(struct arm_lpae_io_pgtable *data, 742 742 + struct io_pgtable_walk_data *walk_data, 743 743 + arm_lpae_iopte *ptep, int lvl) 744 744 + { 745 745 + struct io_pgtable *iop = &data->iop; 746 746 + arm_lpae_iopte pte = READ_ONCE(*ptep); 747 747 + 748 748 + if (iopte_leaf(pte, lvl, iop->fmt)) { 749 749 + size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data); 750 750 + 751 751 + if (iopte_writeable_dirty(pte)) { 752 752 + iommu_dirty_bitmap_record(walk_data->dirty, 753 753 + walk_data->addr, size); 754 754 + if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR)) 755 755 + iopte_set_writeable_clean(ptep); 756 756 + } 757 757 + walk_data->addr += size; 758 758 + return 0; 759 759 + } 760 760 + 761 761 + if (WARN_ON(!iopte_table(pte, lvl))) 762 762 + return -EINVAL; 763 763 + 764 764 + ptep = iopte_deref(pte, data); 765 765 + return __arm_lpae_iopte_walk_dirty(data, walk_data, ptep, lvl + 1); 766 766 + } 767 767 + 768 768 + static int __arm_lpae_iopte_walk_dirty(struct arm_lpae_io_pgtable *data, 769 769 + struct io_pgtable_walk_data *walk_data, 770 770 + arm_lpae_iopte *ptep, 771 771 + int lvl) 772 772 + { 773 773 + u32 idx; 774 774 + int max_entries, ret; 775 775 + 776 776 + if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 777 777 + return -EINVAL; 778 778 + 779 779 + if (lvl == data->start_level) 780 780 + max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); 781 781 + else 782 782 + max_entries = ARM_LPAE_PTES_PER_TABLE(data); 783 783 + 784 784 + for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data); 785 785 + (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) { 786 786 + ret = io_pgtable_visit_dirty(data, walk_data, ptep + idx, lvl); 787 787 + if (ret) 788 788 + return ret; 789 789 + } 790 790 + 791 791 + return 0; 792 792 + } 793 793 + 794 794 + static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops, 795 795 + unsigned long iova, size_t size, 796 796 + unsigned long flags, 797 797 + struct iommu_dirty_bitmap *dirty) 798 798 + { 799 799 + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 800 800 + struct io_pgtable_cfg *cfg = &data->iop.cfg; 801 801 + struct io_pgtable_walk_data walk_data = { 802 802 + .dirty = dirty, 803 803 + .flags = flags, 804 804 + .addr = iova, 805 805 + .end = iova + size, 806 806 + }; 807 807 + arm_lpae_iopte *ptep = data->pgd; 808 808 + int lvl = data->start_level; 809 809 + 810 810 + if (WARN_ON(!size)) 811 811 + return -EINVAL; 812 812 + if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1))) 813 813 + return -EINVAL; 814 814 + if (data->iop.fmt != ARM_64_LPAE_S1) 815 815 + return -EINVAL; 816 816 + 817 817 + return __arm_lpae_iopte_walk_dirty(data, &walk_data, ptep, lvl); 818 818 + } 819 819 + 749 820 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) 750 821 { 751 822 unsigned long granule, page_sizes; ··· 915 804 .map_pages = arm_lpae_map_pages, 916 805 .unmap_pages = arm_lpae_unmap_pages, 917 806 .iova_to_phys = arm_lpae_iova_to_phys, 807 807 + .read_and_clear_dirty = arm_lpae_read_and_clear_dirty, 918 808 }; 919 809 920 810 return data; ··· 931 819 932 820 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | 933 821 IO_PGTABLE_QUIRK_ARM_TTBR1 | 934 934 - IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) 822 822 + IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | 823 823 + IO_PGTABLE_QUIRK_ARM_HD)) 935 824 return NULL; 936 825 937 826 data = arm_lpae_alloc_pgtable(cfg);

+7

drivers/iommu/iommu-priv.h

reviewed

··· 17 17 return dev->iommu->iommu_dev->ops; 18 18 } 19 19 20 20 + const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode); 21 21 + 22 22 + static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwspec) 23 23 + { 24 24 + return iommu_ops_from_fwnode(fwspec ? fwspec->iommu_fwnode : NULL); 25 25 + } 26 26 + 20 27 int iommu_group_replace_domain(struct iommu_group *group, 21 28 struct iommu_domain *new_domain); 22 29

+4 -2

drivers/iommu/iommu-sva.c

reviewed

··· 10 10 #include "iommu-priv.h" 11 11 12 12 static DEFINE_MUTEX(iommu_sva_lock); 13 13 + static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 14 14 + struct mm_struct *mm); 13 15 14 16 /* Allocate a PASID for the mm within range (inclusive) */ 15 17 static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct device *dev) ··· 293 291 return 0; 294 292 } 295 293 296 296 - struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 297 297 - struct mm_struct *mm) 294 294 + static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 295 295 + struct mm_struct *mm) 298 296 { 299 297 const struct iommu_ops *ops = dev_iommu_ops(dev); 300 298 struct iommu_domain *domain;

+28 -12

drivers/iommu/iommu.c

reviewed

··· 510 510 static int __iommu_probe_device(struct device *dev, struct list_head *group_list) 511 511 { 512 512 const struct iommu_ops *ops; 513 513 - struct iommu_fwspec *fwspec; 514 513 struct iommu_group *group; 515 514 struct group_device *gdev; 516 515 int ret; ··· 522 523 * be present, and that any of their registered instances has suitable 523 524 * ops for probing, and thus cheekily co-opt the same mechanism. 524 525 */ 525 525 - fwspec = dev_iommu_fwspec_get(dev); 526 526 - if (fwspec && fwspec->ops) 527 527 - ops = fwspec->ops; 528 528 - else 529 529 - ops = iommu_ops_from_fwnode(NULL); 530 530 - 526 526 + ops = iommu_fwspec_ops(dev_iommu_fwspec_get(dev)); 531 527 if (!ops) 532 528 return -ENODEV; 533 529 /* ··· 2010 2016 return 0; 2011 2017 } 2012 2018 2019 2019 + /* 2020 2020 + * The iommu ops in bus has been retired. Do not use this interface in 2021 2021 + * new drivers. 2022 2022 + */ 2013 2023 struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) 2014 2024 { 2015 2025 const struct iommu_ops *ops = NULL; ··· 2029 2031 return domain; 2030 2032 } 2031 2033 EXPORT_SYMBOL_GPL(iommu_domain_alloc); 2034 2034 + 2035 2035 + /** 2036 2036 + * iommu_paging_domain_alloc() - Allocate a paging domain 2037 2037 + * @dev: device for which the domain is allocated 2038 2038 + * 2039 2039 + * Allocate a paging domain which will be managed by a kernel driver. Return 2040 2040 + * allocated domain if successful, or a ERR pointer for failure. 2041 2041 + */ 2042 2042 + struct iommu_domain *iommu_paging_domain_alloc(struct device *dev) 2043 2043 + { 2044 2044 + if (!dev_has_iommu(dev)) 2045 2045 + return ERR_PTR(-ENODEV); 2046 2046 + 2047 2047 + return __iommu_domain_alloc(dev_iommu_ops(dev), dev, IOMMU_DOMAIN_UNMANAGED); 2048 2048 + } 2049 2049 + EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc); 2032 2050 2033 2051 void iommu_domain_free(struct iommu_domain *domain) 2034 2052 { ··· 2836 2822 return ops; 2837 2823 } 2838 2824 2839 2839 - int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 2840 2840 - const struct iommu_ops *ops) 2825 2825 + int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode) 2841 2826 { 2827 2827 + const struct iommu_ops *ops = iommu_ops_from_fwnode(iommu_fwnode); 2842 2828 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2843 2829 2830 2830 + if (!ops) 2831 2831 + return -EPROBE_DEFER; 2832 2832 + 2844 2833 if (fwspec) 2845 2845 - return ops == fwspec->ops ? 0 : -EINVAL; 2834 2834 + return ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL; 2846 2835 2847 2836 if (!dev_iommu_get(dev)) 2848 2837 return -ENOMEM; ··· 2855 2838 if (!fwspec) 2856 2839 return -ENOMEM; 2857 2840 2858 2858 - of_node_get(to_of_node(iommu_fwnode)); 2841 2841 + fwnode_handle_get(iommu_fwnode); 2859 2842 fwspec->iommu_fwnode = iommu_fwnode; 2860 2860 - fwspec->ops = ops; 2861 2843 dev_iommu_fwspec_set(dev, fwspec); 2862 2844 return 0; 2863 2845 }

+7 -3

drivers/iommu/iommufd/hw_pagetable.c

reviewed

··· 119 119 return ERR_PTR(-EOPNOTSUPP); 120 120 if (flags & ~valid_flags) 121 121 return ERR_PTR(-EOPNOTSUPP); 122 122 + if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) && 123 123 + !device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING)) 124 124 + return ERR_PTR(-EOPNOTSUPP); 122 125 123 126 hwpt_paging = __iommufd_object_alloc( 124 127 ictx, hwpt_paging, IOMMUFD_OBJ_HWPT_PAGING, common.obj); ··· 145 142 } 146 143 hwpt->domain->owner = ops; 147 144 } else { 148 148 - hwpt->domain = iommu_domain_alloc(idev->dev->bus); 149 149 - if (!hwpt->domain) { 150 150 - rc = -ENOMEM; 145 145 + hwpt->domain = iommu_paging_domain_alloc(idev->dev); 146 146 + if (IS_ERR(hwpt->domain)) { 147 147 + rc = PTR_ERR(hwpt->domain); 148 148 + hwpt->domain = NULL; 151 149 goto out_abort; 152 150 } 153 151 }

+4 -3

drivers/iommu/iommufd/pages.c

reviewed

··· 809 809 810 810 lock_limit = task_rlimit(pages->source_task, RLIMIT_MEMLOCK) >> 811 811 PAGE_SHIFT; 812 812 + 813 813 + cur_pages = atomic_long_read(&pages->source_user->locked_vm); 812 814 do { 813 813 - cur_pages = atomic_long_read(&pages->source_user->locked_vm); 814 815 new_pages = cur_pages + npages; 815 816 if (new_pages > lock_limit) 816 817 return -ENOMEM; 817 817 - } while (atomic_long_cmpxchg(&pages->source_user->locked_vm, cur_pages, 818 818 - new_pages) != cur_pages); 818 818 + } while (!atomic_long_try_cmpxchg(&pages->source_user->locked_vm, 819 819 + &cur_pages, new_pages)); 819 820 return 0; 820 821 } 821 822

+1

drivers/iommu/iova.c

reviewed

··· 1000 1000 EXPORT_SYMBOL_GPL(iova_cache_put); 1001 1001 1002 1002 MODULE_AUTHOR("Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>"); 1003 1003 + MODULE_DESCRIPTION("IOMMU I/O Virtual Address management"); 1003 1004 MODULE_LICENSE("GPL");

+3 -9

drivers/iommu/mtk_iommu_v1.c

reviewed

··· 401 401 static int mtk_iommu_v1_create_mapping(struct device *dev, 402 402 const struct of_phandle_args *args) 403 403 { 404 404 - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 405 404 struct mtk_iommu_v1_data *data; 406 405 struct platform_device *m4updev; 407 406 struct dma_iommu_mapping *mtk_mapping; ··· 412 413 return -EINVAL; 413 414 } 414 415 415 415 - if (!fwspec) { 416 416 - ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_v1_ops); 417 417 - if (ret) 418 418 - return ret; 419 419 - fwspec = dev_iommu_fwspec_get(dev); 420 420 - } else if (dev_iommu_fwspec_get(dev)->ops != &mtk_iommu_v1_ops) { 421 421 - return -EINVAL; 422 422 - } 416 416 + ret = iommu_fwspec_init(dev, of_fwnode_handle(args->np)); 417 417 + if (ret) 418 418 + return ret; 423 419 424 420 if (!dev_iommu_priv_get(dev)) { 425 421 /* Get the m4u device */

+27 -32

drivers/iommu/of_iommu.c

reviewed

··· 17 17 #include <linux/slab.h> 18 18 #include <linux/fsl/mc.h> 19 19 20 20 + #include "iommu-priv.h" 21 21 + 20 22 static int of_iommu_xlate(struct device *dev, 21 23 struct of_phandle_args *iommu_spec) 22 24 { 23 25 const struct iommu_ops *ops; 24 24 - struct fwnode_handle *fwnode = &iommu_spec->np->fwnode; 25 26 int ret; 26 27 27 27 - ops = iommu_ops_from_fwnode(fwnode); 28 28 - if ((ops && !ops->of_xlate) || 29 29 - !of_device_is_available(iommu_spec->np)) 28 28 + if (!of_device_is_available(iommu_spec->np)) 30 29 return -ENODEV; 31 30 32 32 - ret = iommu_fwspec_init(dev, fwnode, ops); 31 31 + ret = iommu_fwspec_init(dev, of_fwnode_handle(iommu_spec->np)); 32 32 + if (ret == -EPROBE_DEFER) 33 33 + return driver_deferred_probe_check_state(dev); 33 34 if (ret) 34 35 return ret; 35 35 - /* 36 36 - * The otherwise-empty fwspec handily serves to indicate the specific 37 37 - * IOMMU device we're waiting for, which will be useful if we ever get 38 38 - * a proper probe-ordering dependency mechanism in future. 39 39 - */ 40 40 - if (!ops) 41 41 - return driver_deferred_probe_check_state(dev); 42 36 43 43 - if (!try_module_get(ops->owner)) 37 37 + ops = iommu_ops_from_fwnode(&iommu_spec->np->fwnode); 38 38 + if (!ops->of_xlate || !try_module_get(ops->owner)) 44 39 return -ENODEV; 45 40 46 41 ret = ops->of_xlate(dev, iommu_spec); ··· 100 105 of_iommu_configure_dev(master_np, dev); 101 106 } 102 107 108 108 + static void of_pci_check_device_ats(struct device *dev, struct device_node *np) 109 109 + { 110 110 + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 111 111 + 112 112 + if (fwspec && of_property_read_bool(np, "ats-supported")) 113 113 + fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS; 114 114 + } 115 115 + 103 116 /* 104 117 * Returns: 105 118 * 0 on success, an iommu was configured ··· 118 115 int of_iommu_configure(struct device *dev, struct device_node *master_np, 119 116 const u32 *id) 120 117 { 121 121 - struct iommu_fwspec *fwspec; 122 118 int err; 123 119 124 120 if (!master_np) ··· 125 123 126 124 /* Serialise to make dev->iommu stable under our potential fwspec */ 127 125 mutex_lock(&iommu_probe_device_lock); 128 128 - fwspec = dev_iommu_fwspec_get(dev); 129 129 - if (fwspec) { 130 130 - if (fwspec->ops) { 131 131 - mutex_unlock(&iommu_probe_device_lock); 132 132 - return 0; 133 133 - } 134 134 - /* In the deferred case, start again from scratch */ 135 135 - iommu_fwspec_free(dev); 126 126 + if (dev_iommu_fwspec_get(dev)) { 127 127 + mutex_unlock(&iommu_probe_device_lock); 128 128 + return 0; 136 129 } 137 130 138 131 /* ··· 144 147 pci_request_acs(); 145 148 err = pci_for_each_dma_alias(to_pci_dev(dev), 146 149 of_pci_iommu_init, &info); 150 150 + of_pci_check_device_ats(dev, master_np); 147 151 } else { 148 152 err = of_iommu_configure_device(master_np, dev, id); 149 153 } 154 154 + 155 155 + if (err) 156 156 + iommu_fwspec_free(dev); 150 157 mutex_unlock(&iommu_probe_device_lock); 151 158 152 152 - if (err == -ENODEV || err == -EPROBE_DEFER) 153 153 - return err; 154 154 - if (err) 155 155 - goto err_log; 159 159 + if (!err && dev->bus) 160 160 + err = iommu_probe_device(dev); 156 161 157 157 - err = iommu_probe_device(dev); 158 158 - if (err) 159 159 - goto err_log; 160 160 - return 0; 162 162 + if (err && err != -EPROBE_DEFER) 163 163 + dev_dbg(dev, "Adding to IOMMU failed: %d\n", err); 161 164 162 162 - err_log: 163 163 - dev_dbg(dev, "Adding to IOMMU failed: %pe\n", ERR_PTR(err)); 164 165 return err; 165 166 } 166 167

+13 -2

drivers/iommu/sun50i-iommu.c

reviewed

··· 452 452 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 453 453 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 454 454 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 455 455 + iommu_write(iommu, IOMMU_BYPASS_REG, 0); 455 456 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 456 457 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 457 458 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | ··· 602 601 u32 *page_table, *pte_addr; 603 602 int ret = 0; 604 603 604 604 + /* the IOMMU can only handle 32-bit addresses, both input and output */ 605 605 + if ((uint64_t)paddr >> 32) { 606 606 + ret = -EINVAL; 607 607 + dev_warn_once(iommu->dev, 608 608 + "attempt to map address beyond 4GB\n"); 609 609 + goto out; 610 610 + } 611 611 + 605 612 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 606 613 if (IS_ERR(page_table)) { 607 614 ret = PTR_ERR(page_table); ··· 690 681 if (!sun50i_domain) 691 682 return NULL; 692 683 693 693 - sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL, get_order(DT_SIZE)); 684 684 + sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, 685 685 + get_order(DT_SIZE)); 694 686 if (!sun50i_domain->dt) 695 687 goto err_free_domain; 696 688 ··· 1006 996 1007 997 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 1008 998 PT_SIZE, PT_SIZE, 1009 1009 - SLAB_HWCACHE_ALIGN, 999 999 + SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA32, 1010 1000 NULL); 1011 1001 if (!iommu->pt_pool) 1012 1002 return -ENOMEM; ··· 1067 1057 1068 1058 static const struct of_device_id sun50i_iommu_dt[] = { 1069 1059 { .compatible = "allwinner,sun50i-h6-iommu", }, 1060 1060 + { .compatible = "allwinner,sun50i-h616-iommu", }, 1070 1061 { /* sentinel */ }, 1071 1062 }; 1072 1063 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);

+1 -1

drivers/iommu/tegra-smmu.c

reviewed

··· 837 837 const struct iommu_ops *ops = smmu->iommu.ops; 838 838 int err; 839 839 840 840 - err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); 840 840 + err = iommu_fwspec_init(dev, dev_fwnode(smmu->dev)); 841 841 if (err < 0) { 842 842 dev_err(dev, "failed to initialize fwspec: %d\n", err); 843 843 return err;

+3 -3

drivers/net/wireless/ath/ath10k/snoc.c

reviewed

··· 1635 1635 1636 1636 ar_snoc->fw.dev = &pdev->dev; 1637 1637 1638 1638 - iommu_dom = iommu_domain_alloc(&platform_bus_type); 1639 1639 - if (!iommu_dom) { 1638 1638 + iommu_dom = iommu_paging_domain_alloc(ar_snoc->fw.dev); 1639 1639 + if (IS_ERR(iommu_dom)) { 1640 1640 ath10k_err(ar, "failed to allocate iommu domain\n"); 1641 1641 - ret = -ENOMEM; 1641 1641 + ret = PTR_ERR(iommu_dom); 1642 1642 goto err_unregister; 1643 1643 } 1644 1644

+3 -3

drivers/net/wireless/ath/ath11k/ahb.c

reviewed

··· 1031 1031 1032 1032 ab_ahb->fw.dev = &pdev->dev; 1033 1033 1034 1034 - iommu_dom = iommu_domain_alloc(&platform_bus_type); 1035 1035 - if (!iommu_dom) { 1034 1034 + iommu_dom = iommu_paging_domain_alloc(ab_ahb->fw.dev); 1035 1035 + if (IS_ERR(iommu_dom)) { 1036 1036 ath11k_err(ab, "failed to allocate iommu domain\n"); 1037 1037 - ret = -ENOMEM; 1037 1037 + ret = PTR_ERR(iommu_dom); 1038 1038 goto err_unregister; 1039 1039 } 1040 1040

+11 -19

drivers/of/device.c

reviewed

··· 96 96 const struct bus_dma_region *map = NULL; 97 97 struct device_node *bus_np; 98 98 u64 mask, end = 0; 99 99 - bool coherent; 100 100 - int iommu_ret; 99 99 + bool coherent, set_map = false; 101 100 int ret; 102 101 103 102 if (np == dev->of_node) ··· 117 118 } else { 118 119 /* Determine the overall bounds of all DMA regions */ 119 120 end = dma_range_map_max(map); 121 121 + set_map = true; 120 122 } 121 123 122 124 /* ··· 144 144 dev->coherent_dma_mask &= mask; 145 145 *dev->dma_mask &= mask; 146 146 /* ...but only set bus limit and range map if we found valid dma-ranges earlier */ 147 147 - if (!ret) { 147 147 + if (set_map) { 148 148 dev->bus_dma_limit = end; 149 149 dev->dma_range_map = map; 150 150 } ··· 153 153 dev_dbg(dev, "device is%sdma coherent\n", 154 154 coherent ? " " : " not "); 155 155 156 156 - iommu_ret = of_iommu_configure(dev, np, id); 157 157 - if (iommu_ret == -EPROBE_DEFER) { 156 156 + ret = of_iommu_configure(dev, np, id); 157 157 + if (ret == -EPROBE_DEFER) { 158 158 /* Don't touch range map if it wasn't set from a valid dma-ranges */ 159 159 - if (!ret) 159 159 + if (set_map) 160 160 dev->dma_range_map = NULL; 161 161 kfree(map); 162 162 return -EPROBE_DEFER; 163 163 - } else if (iommu_ret == -ENODEV) { 164 164 - dev_dbg(dev, "device is not behind an iommu\n"); 165 165 - } else if (iommu_ret) { 166 166 - dev_err(dev, "iommu configuration for device failed with %pe\n", 167 167 - ERR_PTR(iommu_ret)); 168 168 - 169 169 - /* 170 170 - * Historically this routine doesn't fail driver probing 171 171 - * due to errors in of_iommu_configure() 172 172 - */ 173 173 - } else 174 174 - dev_dbg(dev, "device is behind an iommu\n"); 163 163 + } 164 164 + /* Take all other IOMMU errors to mean we'll just carry on without it */ 165 165 + dev_dbg(dev, "device is%sbehind an iommu\n", 166 166 + !ret ? " " : " not "); 175 167 176 168 arch_setup_dma_ops(dev, coherent); 177 169 178 178 - if (iommu_ret) 170 170 + if (ret) 179 171 of_dma_set_restricted_buffer(dev, np); 180 172 181 173 return 0;

+4 -3

drivers/vfio/vfio_iommu_type1.c

reviewed

··· 2135 2135 { 2136 2136 struct iommu_domain **domain = data; 2137 2137 2138 2138 - *domain = iommu_domain_alloc(dev->bus); 2138 2138 + *domain = iommu_paging_domain_alloc(dev); 2139 2139 return 1; /* Don't iterate */ 2140 2140 } 2141 2141 ··· 2192 2192 * us a representative device for the IOMMU API call. We don't actually 2193 2193 * want to iterate beyond the first device (if any). 2194 2194 */ 2195 2195 - ret = -EIO; 2196 2195 iommu_group_for_each_dev(iommu_group, &domain->domain, 2197 2196 vfio_iommu_domain_alloc); 2198 2198 - if (!domain->domain) 2197 2197 + if (IS_ERR(domain->domain)) { 2198 2198 + ret = PTR_ERR(domain->domain); 2199 2199 goto out_free_domain; 2200 2200 + } 2200 2201 2201 2202 if (iommu->nesting) { 2202 2203 ret = iommu_enable_nesting(domain->domain);

+6 -8

drivers/vhost/vdpa.c

reviewed

··· 1312 1312 struct vdpa_device *vdpa = v->vdpa; 1313 1313 const struct vdpa_config_ops *ops = vdpa->config; 1314 1314 struct device *dma_dev = vdpa_get_dma_dev(vdpa); 1315 1315 - const struct bus_type *bus; 1316 1315 int ret; 1317 1316 1318 1317 /* Device want to do DMA by itself */ 1319 1318 if (ops->set_map || ops->dma_map) 1320 1319 return 0; 1321 1321 - 1322 1322 - bus = dma_dev->bus; 1323 1323 - if (!bus) 1324 1324 - return -EFAULT; 1325 1320 1326 1321 if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) { 1327 1322 dev_warn_once(&v->dev, ··· 1324 1329 return -ENOTSUPP; 1325 1330 } 1326 1331 1327 1327 - v->domain = iommu_domain_alloc(bus); 1328 1328 - if (!v->domain) 1329 1329 - return -EIO; 1332 1332 + v->domain = iommu_paging_domain_alloc(dma_dev); 1333 1333 + if (IS_ERR(v->domain)) { 1334 1334 + ret = PTR_ERR(v->domain); 1335 1335 + v->domain = NULL; 1336 1336 + return ret; 1337 1337 + } 1330 1338 1331 1339 ret = iommu_attach_device(v->domain, dma_dev); 1332 1340 if (ret)

+1 -2

include/acpi/acpi_bus.h

reviewed

··· 737 737 bool acpi_dma_supported(const struct acpi_device *adev); 738 738 enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev); 739 739 int acpi_iommu_fwspec_init(struct device *dev, u32 id, 740 740 - struct fwnode_handle *fwnode, 741 741 - const struct iommu_ops *ops); 740 740 + struct fwnode_handle *fwnode); 742 741 int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map); 743 742 int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, 744 743 const u32 *input_id);

+3

include/linux/io-pgtable.h

reviewed

··· 85 85 * 86 86 * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability 87 87 * attributes set in the TCR for a non-coherent page-table walker. 88 88 + * 89 89 + * IO_PGTABLE_QUIRK_ARM_HD: Enables dirty tracking in stage 1 pagetable. 88 90 */ 89 91 #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) 90 92 #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) ··· 94 92 #define IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT BIT(4) 95 93 #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) 96 94 #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) 95 95 + #define IO_PGTABLE_QUIRK_ARM_HD BIT(7) 97 96 unsigned long quirks; 98 97 unsigned long pgsize_bitmap; 99 98 unsigned int ias;

+11 -24

include/linux/iommu.h

reviewed

··· 321 321 #define IOMMU_PASID_INVALID (-1U) 322 322 typedef unsigned int ioasid_t; 323 323 324 324 + /* Read but do not clear any dirty bits */ 325 325 + #define IOMMU_DIRTY_NO_CLEAR (1 << 0) 326 326 + 324 327 #ifdef CONFIG_IOMMU_API 325 328 326 329 /** ··· 359 356 struct iova_bitmap *bitmap; 360 357 struct iommu_iotlb_gather *gather; 361 358 }; 362 362 - 363 363 - /* Read but do not clear any dirty bits */ 364 364 - #define IOMMU_DIRTY_NO_CLEAR (1 << 0) 365 359 366 360 /** 367 361 * struct iommu_dirty_ops - domain specific dirty tracking operations ··· 789 789 extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); 790 790 extern bool iommu_group_has_isolated_msi(struct iommu_group *group); 791 791 extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus); 792 792 + struct iommu_domain *iommu_paging_domain_alloc(struct device *dev); 792 793 extern void iommu_domain_free(struct iommu_domain *domain); 793 794 extern int iommu_attach_device(struct iommu_domain *domain, 794 795 struct device *dev); ··· 978 977 979 978 /** 980 979 * struct iommu_fwspec - per-device IOMMU instance data 981 981 - * @ops: ops for this device's IOMMU 982 980 * @iommu_fwnode: firmware handle for this device's IOMMU 983 981 * @flags: IOMMU_FWSPEC_* flags 984 982 * @num_ids: number of associated device IDs ··· 988 988 * consumers. 989 989 */ 990 990 struct iommu_fwspec { 991 991 - const struct iommu_ops *ops; 992 991 struct fwnode_handle *iommu_fwnode; 993 992 u32 flags; 994 993 unsigned int num_ids; ··· 1021 1022 struct list_head sva_domains; 1022 1023 }; 1023 1024 1024 1024 - int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 1025 1025 - const struct iommu_ops *ops); 1025 1025 + int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode); 1026 1026 void iommu_fwspec_free(struct device *dev); 1027 1027 int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); 1028 1028 - const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode); 1029 1028 1030 1029 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 1031 1030 { ··· 1096 1099 static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) 1097 1100 { 1098 1101 return NULL; 1102 1102 + } 1103 1103 + 1104 1104 + static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev) 1105 1105 + { 1106 1106 + return ERR_PTR(-ENODEV); 1099 1107 } 1100 1108 1101 1109 static inline void iommu_domain_free(struct iommu_domain *domain) ··· 1332 1330 } 1333 1331 1334 1332 static inline int iommu_fwspec_init(struct device *dev, 1335 1335 - struct fwnode_handle *iommu_fwnode, 1336 1336 - const struct iommu_ops *ops) 1333 1333 + struct fwnode_handle *iommu_fwnode) 1337 1334 { 1338 1335 return -ENODEV; 1339 1336 } ··· 1345 1344 int num_ids) 1346 1345 { 1347 1346 return -ENODEV; 1348 1348 - } 1349 1349 - 1350 1350 - static inline 1351 1351 - const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) 1352 1352 - { 1353 1353 - return NULL; 1354 1347 } 1355 1348 1356 1349 static inline int ··· 1531 1536 struct mm_struct *mm); 1532 1537 void iommu_sva_unbind_device(struct iommu_sva *handle); 1533 1538 u32 iommu_sva_get_pasid(struct iommu_sva *handle); 1534 1534 - struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 1535 1535 - struct mm_struct *mm); 1536 1539 #else 1537 1540 static inline struct iommu_sva * 1538 1541 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) ··· 1555 1562 } 1556 1563 1557 1564 static inline void mm_pasid_drop(struct mm_struct *mm) {} 1558 1558 - 1559 1559 - static inline struct iommu_domain * 1560 1560 - iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) 1561 1561 - { 1562 1562 - return NULL; 1563 1563 - } 1564 1565 #endif /* CONFIG_IOMMU_SVA */ 1565 1566 1566 1567 #ifdef CONFIG_IOMMU_IOPF