tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Merge tag 'dma-mapping-6.18-2025-09-30' of git://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux

Pull dma-mapping updates from Marek Szyprowski:

- Refactoring of DMA mapping API to physical addresses as the primary
interface instead of page+offset parameters

This gets much closer to Matthew Wilcox's long term wish for
struct-pageless IO to cacheable DRAM and is supporting memdesc
project which seeks to substantially transform how struct page works.

An advantage of this approach is the possibility of introducing
DMA_ATTR_MMIO, which covers existing 'dma_map_resource' flow in the
common paths, what in turn lets to use recently introduced
dma_iova_link() API to map PCI P2P MMIO without creating struct page

Developped by Leon Romanovsky and Jason Gunthorpe

- Minor clean-up by Petr Tesarik and Qianfeng Rong

* tag 'dma-mapping-6.18-2025-09-30' of git://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux:
kmsan: fix missed kmsan_handle_dma() signature conversion
mm/hmm: properly take MMIO path
mm/hmm: migrate to physical address-based DMA mapping API
dma-mapping: export new dma_*map_phys() interface
xen: swiotlb: Open code map_resource callback
dma-mapping: implement DMA_ATTR_MMIO for dma_(un)map_page_attrs()
kmsan: convert kmsan_handle_dma to use physical addresses
dma-mapping: convert dma_direct_*map_page to be phys_addr_t based
iommu/dma: implement DMA_ATTR_MMIO for iommu_dma_(un)map_phys()
iommu/dma: rename iommu_dma_*map_page to iommu_dma_*map_phys
dma-mapping: rename trace_dma_*map_page to trace_dma_*map_phys
dma-debug: refactor to use physical addresses for page mapping
iommu/dma: implement DMA_ATTR_MMIO for dma_iova_link().
dma-mapping: introduce new DMA attribute to indicate MMIO memory
swiotlb: Remove redundant __GFP_NOWARN
dma-direct: clean up the logic in __dma_direct_alloc_pages()

Linus Torvalds a498d59c ee2fe81c

+294 -275
+2 -2
Documentation/core-api/dma-api.rst
··· 761 761 [<ffffffff80235177>] find_busiest_group+0x207/0x8a0 762 762 [<ffffffff8064784f>] _spin_lock_irqsave+0x1f/0x50 763 763 [<ffffffff803c7ea3>] check_unmap+0x203/0x490 764 - [<ffffffff803c8259>] debug_dma_unmap_page+0x49/0x50 764 + [<ffffffff803c8259>] debug_dma_unmap_phys+0x49/0x50 765 765 [<ffffffff80485f26>] nv_tx_done_optimized+0xc6/0x2c0 766 766 [<ffffffff80486c13>] nv_nic_irq_optimized+0x73/0x2b0 767 767 [<ffffffff8026df84>] handle_IRQ_event+0x34/0x70 ··· 855 855 dma-debug interface debug_dma_mapping_error() to debug drivers that fail 856 856 to check DMA mapping errors on addresses returned by dma_map_single() and 857 857 dma_map_page() interfaces. This interface clears a flag set by 858 - debug_dma_map_page() to indicate that dma_mapping_error() has been called by 858 + debug_dma_map_phys() to indicate that dma_mapping_error() has been called by 859 859 the driver. When driver does unmap, debug_dma_unmap() checks the flag and if 860 860 this flag is still set, prints warning message that includes call trace that 861 861 leads up to the unmap. This interface can be called from dma_mapping_error()
+18
Documentation/core-api/dma-attributes.rst
··· 130 130 subsystem that the buffer is fully accessible at the elevated privilege 131 131 level (and ideally inaccessible or at least read-only at the 132 132 lesser-privileged levels). 133 + 134 + DMA_ATTR_MMIO 135 + ------------- 136 + 137 + This attribute indicates the physical address is not normal system 138 + memory. It may not be used with kmap*()/phys_to_virt()/phys_to_page() 139 + functions, it may not be cacheable, and access using CPU load/store 140 + instructions may not be allowed. 141 + 142 + Usually this will be used to describe MMIO addresses, or other non-cacheable 143 + register addresses. When DMA mapping this sort of address we call 144 + the operation Peer to Peer as a one device is DMA'ing to another device. 145 + For PCI devices the p2pdma APIs must be used to determine if 146 + DMA_ATTR_MMIO is appropriate. 147 + 148 + For architectures that require cache flushing for DMA coherence 149 + DMA_ATTR_MMIO will not perform any cache flushing. The address 150 + provided must never be mapped cacheable into the CPU.
+2 -2
arch/powerpc/kernel/dma-iommu.c
··· 14 14 #define can_map_direct(dev, addr) \ 15 15 ((dev)->bus_dma_limit >= phys_to_dma((dev), (addr))) 16 16 17 - bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr) 17 + bool arch_dma_map_phys_direct(struct device *dev, phys_addr_t addr) 18 18 { 19 19 if (likely(!dev->bus_dma_limit)) 20 20 return false; ··· 24 24 25 25 #define is_direct_handle(dev, h) ((h) >= (dev)->archdata.dma_offset) 26 26 27 - bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle) 27 + bool arch_dma_unmap_phys_direct(struct device *dev, dma_addr_t dma_handle) 28 28 { 29 29 if (likely(!dev->bus_dma_limit)) 30 30 return false;
+31 -30
drivers/iommu/dma-iommu.c
··· 724 724 static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, 725 725 unsigned long attrs) 726 726 { 727 - int prot = coherent ? IOMMU_CACHE : 0; 727 + int prot; 728 + 729 + if (attrs & DMA_ATTR_MMIO) 730 + prot = IOMMU_MMIO; 731 + else 732 + prot = coherent ? IOMMU_CACHE : 0; 728 733 729 734 if (attrs & DMA_ATTR_PRIVILEGED) 730 735 prot |= IOMMU_PRIV; ··· 1195 1190 return iova_offset(iovad, phys | size); 1196 1191 } 1197 1192 1198 - dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 1199 - unsigned long offset, size_t size, enum dma_data_direction dir, 1200 - unsigned long attrs) 1193 + dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 1194 + enum dma_data_direction dir, unsigned long attrs) 1201 1195 { 1202 - phys_addr_t phys = page_to_phys(page) + offset; 1203 1196 bool coherent = dev_is_dma_coherent(dev); 1204 1197 int prot = dma_info_to_prot(dir, coherent, attrs); 1205 1198 struct iommu_domain *domain = iommu_get_dma_domain(dev); ··· 1211 1208 */ 1212 1209 if (dev_use_swiotlb(dev, size, dir) && 1213 1210 iova_unaligned(iovad, phys, size)) { 1211 + if (attrs & DMA_ATTR_MMIO) 1212 + return DMA_MAPPING_ERROR; 1213 + 1214 1214 phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs); 1215 1215 if (phys == (phys_addr_t)DMA_MAPPING_ERROR) 1216 1216 return DMA_MAPPING_ERROR; 1217 1217 } 1218 1218 1219 - if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1219 + if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) 1220 1220 arch_sync_dma_for_device(phys, size, dir); 1221 1221 1222 1222 iova = __iommu_dma_map(dev, phys, size, prot, dma_mask); 1223 - if (iova == DMA_MAPPING_ERROR) 1223 + if (iova == DMA_MAPPING_ERROR && !(attrs & DMA_ATTR_MMIO)) 1224 1224 swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); 1225 1225 return iova; 1226 1226 } 1227 1227 1228 - void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 1228 + void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle, 1229 1229 size_t size, enum dma_data_direction dir, unsigned long attrs) 1230 1230 { 1231 - struct iommu_domain *domain = iommu_get_dma_domain(dev); 1232 1231 phys_addr_t phys; 1233 1232 1234 - phys = iommu_iova_to_phys(domain, dma_handle); 1233 + if (attrs & DMA_ATTR_MMIO) { 1234 + __iommu_dma_unmap(dev, dma_handle, size); 1235 + return; 1236 + } 1237 + 1238 + phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); 1235 1239 if (WARN_ON(!phys)) 1236 1240 return; 1237 1241 ··· 1351 1341 int i; 1352 1342 1353 1343 for_each_sg(sg, s, nents, i) 1354 - iommu_dma_unmap_page(dev, sg_dma_address(s), 1344 + iommu_dma_unmap_phys(dev, sg_dma_address(s), 1355 1345 sg_dma_len(s), dir, attrs); 1356 1346 } 1357 1347 ··· 1364 1354 sg_dma_mark_swiotlb(sg); 1365 1355 1366 1356 for_each_sg(sg, s, nents, i) { 1367 - sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s), 1368 - s->offset, s->length, dir, attrs); 1357 + sg_dma_address(s) = iommu_dma_map_phys(dev, sg_phys(s), 1358 + s->length, dir, attrs); 1369 1359 if (sg_dma_address(s) == DMA_MAPPING_ERROR) 1370 1360 goto out_unmap; 1371 1361 sg_dma_len(s) = s->length; ··· 1554 1544 1555 1545 if (end) 1556 1546 __iommu_dma_unmap(dev, start, end - start); 1557 - } 1558 - 1559 - dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, 1560 - size_t size, enum dma_data_direction dir, unsigned long attrs) 1561 - { 1562 - return __iommu_dma_map(dev, phys, size, 1563 - dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, 1564 - dma_get_mask(dev)); 1565 - } 1566 - 1567 - void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, 1568 - size_t size, enum dma_data_direction dir, unsigned long attrs) 1569 - { 1570 - __iommu_dma_unmap(dev, handle, size); 1571 1547 } 1572 1548 1573 1549 static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr) ··· 1834 1838 unsigned long attrs) 1835 1839 { 1836 1840 bool coherent = dev_is_dma_coherent(dev); 1841 + int prot = dma_info_to_prot(dir, coherent, attrs); 1837 1842 1838 - if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1843 + if (!coherent && !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) 1839 1844 arch_sync_dma_for_device(phys, size, dir); 1840 1845 1841 1846 return iommu_map_nosync(iommu_get_dma_domain(dev), addr, phys, size, 1842 - dma_info_to_prot(dir, coherent, attrs), GFP_ATOMIC); 1847 + prot, GFP_ATOMIC); 1843 1848 } 1844 1849 1845 1850 static int iommu_dma_iova_bounce_and_link(struct device *dev, dma_addr_t addr, ··· 1946 1949 return -EIO; 1947 1950 1948 1951 if (dev_use_swiotlb(dev, size, dir) && 1949 - iova_unaligned(iovad, phys, size)) 1952 + iova_unaligned(iovad, phys, size)) { 1953 + if (attrs & DMA_ATTR_MMIO) 1954 + return -EPERM; 1955 + 1950 1956 return iommu_dma_iova_link_swiotlb(dev, state, phys, offset, 1951 1957 size, dir, attrs); 1958 + } 1952 1959 1953 1960 return __dma_iova_link(dev, state->addr + offset - iova_start_pad, 1954 1961 phys - iova_start_pad,
+2 -2
drivers/virtio/virtio_ring.c
··· 378 378 * is initialized by the hardware. Explicitly check/unpoison it 379 379 * depending on the direction. 380 380 */ 381 - kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction); 381 + kmsan_handle_dma(sg_phys(sg), sg->length, direction); 382 382 *addr = (dma_addr_t)sg_phys(sg); 383 383 return 0; 384 384 } ··· 3157 3157 struct vring_virtqueue *vq = to_vvq(_vq); 3158 3158 3159 3159 if (!vq->use_dma_api) { 3160 - kmsan_handle_dma(virt_to_page(ptr), offset_in_page(ptr), size, dir); 3160 + kmsan_handle_dma(virt_to_phys(ptr), size, dir); 3161 3161 return (dma_addr_t)virt_to_phys(ptr); 3162 3162 } 3163 3163
+20 -1
drivers/xen/swiotlb-xen.c
··· 392 392 } 393 393 } 394 394 395 + static dma_addr_t xen_swiotlb_direct_map_resource(struct device *dev, 396 + phys_addr_t paddr, 397 + size_t size, 398 + enum dma_data_direction dir, 399 + unsigned long attrs) 400 + { 401 + dma_addr_t dma_addr = paddr; 402 + 403 + if (unlikely(!dma_capable(dev, dma_addr, size, false))) { 404 + dev_err_once(dev, 405 + "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 406 + &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 407 + WARN_ON_ONCE(1); 408 + return DMA_MAPPING_ERROR; 409 + } 410 + 411 + return dma_addr; 412 + } 413 + 395 414 /* 396 415 * Return whether the given device DMA address mask can be supported 397 416 * properly. For example, if your device can only drive the low 24-bits ··· 445 426 .alloc_pages_op = dma_common_alloc_pages, 446 427 .free_pages = dma_common_free_pages, 447 428 .max_mapping_size = swiotlb_max_mapping_size, 448 - .map_resource = dma_direct_map_resource, 429 + .map_resource = xen_swiotlb_direct_map_resource, 449 430 };
-2
include/linux/dma-direct.h
··· 149 149 struct page *page, dma_addr_t dma_addr, 150 150 enum dma_data_direction dir); 151 151 int dma_direct_supported(struct device *dev, u64 mask); 152 - dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, 153 - size_t size, enum dma_data_direction dir, unsigned long attrs); 154 152 155 153 #endif /* _LINUX_DMA_DIRECT_H */
+4 -4
include/linux/dma-map-ops.h
··· 395 395 void arch_dma_clear_uncached(void *addr, size_t size); 396 396 397 397 #ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT 398 - bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr); 399 - bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle); 398 + bool arch_dma_map_phys_direct(struct device *dev, phys_addr_t addr); 399 + bool arch_dma_unmap_phys_direct(struct device *dev, dma_addr_t dma_handle); 400 400 bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg, 401 401 int nents); 402 402 bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg, 403 403 int nents); 404 404 #else 405 - #define arch_dma_map_page_direct(d, a) (false) 406 - #define arch_dma_unmap_page_direct(d, a) (false) 405 + #define arch_dma_map_phys_direct(d, a) (false) 406 + #define arch_dma_unmap_phys_direct(d, a) (false) 407 407 #define arch_dma_map_sg_direct(d, s, n) (false) 408 408 #define arch_dma_unmap_sg_direct(d, s, n) (false) 409 409 #endif
+33
include/linux/dma-mapping.h
··· 59 59 #define DMA_ATTR_PRIVILEGED (1UL << 9) 60 60 61 61 /* 62 + * DMA_ATTR_MMIO - Indicates memory-mapped I/O (MMIO) region for DMA mapping 63 + * 64 + * This attribute indicates the physical address is not normal system 65 + * memory. It may not be used with kmap*()/phys_to_virt()/phys_to_page() 66 + * functions, it may not be cacheable, and access using CPU load/store 67 + * instructions may not be allowed. 68 + * 69 + * Usually this will be used to describe MMIO addresses, or other non-cacheable 70 + * register addresses. When DMA mapping this sort of address we call 71 + * the operation Peer to Peer as a one device is DMA'ing to another device. 72 + * For PCI devices the p2pdma APIs must be used to determine if DMA_ATTR_MMIO 73 + * is appropriate. 74 + * 75 + * For architectures that require cache flushing for DMA coherence 76 + * DMA_ATTR_MMIO will not perform any cache flushing. The address 77 + * provided must never be mapped cacheable into the CPU. 78 + */ 79 + #define DMA_ATTR_MMIO (1UL << 10) 80 + 81 + /* 62 82 * A dma_addr_t can hold any valid DMA or bus address for the platform. It can 63 83 * be given to a device to use as a DMA source or target. It is specific to a 64 84 * given device and there may be a translation between the CPU physical address ··· 138 118 unsigned long attrs); 139 119 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 140 120 enum dma_data_direction dir, unsigned long attrs); 121 + dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 122 + enum dma_data_direction dir, unsigned long attrs); 123 + void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, 124 + enum dma_data_direction dir, unsigned long attrs); 141 125 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 142 126 int nents, enum dma_data_direction dir, unsigned long attrs); 143 127 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, ··· 193 169 return DMA_MAPPING_ERROR; 194 170 } 195 171 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, 172 + size_t size, enum dma_data_direction dir, unsigned long attrs) 173 + { 174 + } 175 + static inline dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, 176 + size_t size, enum dma_data_direction dir, unsigned long attrs) 177 + { 178 + return DMA_MAPPING_ERROR; 179 + } 180 + static inline void dma_unmap_phys(struct device *dev, dma_addr_t addr, 196 181 size_t size, enum dma_data_direction dir, unsigned long attrs) 197 182 { 198 183 }
+3 -8
include/linux/iommu-dma.h
··· 21 21 } 22 22 #endif /* CONFIG_IOMMU_DMA */ 23 23 24 - dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 25 - unsigned long offset, size_t size, enum dma_data_direction dir, 26 - unsigned long attrs); 27 - void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, 24 + dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 25 + enum dma_data_direction dir, unsigned long attrs); 26 + void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle, 28 27 size_t size, enum dma_data_direction dir, unsigned long attrs); 29 28 int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 30 29 enum dma_data_direction dir, unsigned long attrs); ··· 42 43 size_t iommu_dma_max_mapping_size(struct device *dev); 43 44 void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, 44 45 dma_addr_t handle, unsigned long attrs); 45 - dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, 46 - size_t size, enum dma_data_direction dir, unsigned long attrs); 47 - void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, 48 - size_t size, enum dma_data_direction dir, unsigned long attrs); 49 46 struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, size_t size, 50 47 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs); 51 48 void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
+4 -5
include/linux/kmsan.h
··· 182 182 183 183 /** 184 184 * kmsan_handle_dma() - Handle a DMA data transfer. 185 - * @page: first page of the buffer. 186 - * @offset: offset of the buffer within the first page. 185 + * @phys: physical address of the buffer. 187 186 * @size: buffer size. 188 187 * @dir: one of possible dma_data_direction values. 189 188 * ··· 191 192 * * initializes the buffer, if it is copied from device; 192 193 * * does both, if this is a DMA_BIDIRECTIONAL transfer. 193 194 */ 194 - void kmsan_handle_dma(struct page *page, size_t offset, size_t size, 195 + void kmsan_handle_dma(phys_addr_t phys, size_t size, 195 196 enum dma_data_direction dir); 196 197 197 198 /** ··· 371 372 { 372 373 } 373 374 374 - static inline void kmsan_handle_dma(struct page *page, size_t offset, 375 - size_t size, enum dma_data_direction dir) 375 + static inline void kmsan_handle_dma(phys_addr_t phys, size_t size, 376 + enum dma_data_direction dir) 376 377 { 377 378 } 378 379
+1
include/linux/page-flags.h
··· 618 618 #else 619 619 PAGEFLAG_FALSE(HighMem, highmem) 620 620 #endif 621 + #define PhysHighMem(__p) (PageHighMem(phys_to_page(__p))) 621 622 622 623 /* Does kmap_local_folio() only allow access to one page of the folio? */ 623 624 #ifdef CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP
+4 -5
include/trace/events/dma.h
··· 31 31 { DMA_ATTR_FORCE_CONTIGUOUS, "FORCE_CONTIGUOUS" }, \ 32 32 { DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \ 33 33 { DMA_ATTR_NO_WARN, "NO_WARN" }, \ 34 - { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }) 34 + { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \ 35 + { DMA_ATTR_MMIO, "MMIO" }) 35 36 36 37 DECLARE_EVENT_CLASS(dma_map, 37 38 TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, ··· 72 71 size_t size, enum dma_data_direction dir, unsigned long attrs), \ 73 72 TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs)) 74 73 75 - DEFINE_MAP_EVENT(dma_map_page); 76 - DEFINE_MAP_EVENT(dma_map_resource); 74 + DEFINE_MAP_EVENT(dma_map_phys); 77 75 78 76 DECLARE_EVENT_CLASS(dma_unmap, 79 77 TP_PROTO(struct device *dev, dma_addr_t addr, size_t size, ··· 109 109 enum dma_data_direction dir, unsigned long attrs), \ 110 110 TP_ARGS(dev, addr, size, dir, attrs)) 111 111 112 - DEFINE_UNMAP_EVENT(dma_unmap_page); 113 - DEFINE_UNMAP_EVENT(dma_unmap_resource); 112 + DEFINE_UNMAP_EVENT(dma_unmap_phys); 114 113 115 114 DECLARE_EVENT_CLASS(dma_alloc_class, 116 115 TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr,
+20 -62
kernel/dma/debug.c
··· 38 38 dma_debug_single, 39 39 dma_debug_sg, 40 40 dma_debug_coherent, 41 - dma_debug_resource, 42 41 dma_debug_noncoherent, 42 + dma_debug_phy, 43 43 }; 44 44 45 45 enum map_err_types { ··· 141 141 [dma_debug_single] = "single", 142 142 [dma_debug_sg] = "scatter-gather", 143 143 [dma_debug_coherent] = "coherent", 144 - [dma_debug_resource] = "resource", 145 144 [dma_debug_noncoherent] = "noncoherent", 145 + [dma_debug_phy] = "phy", 146 146 }; 147 147 148 148 static const char *dir2name[] = { ··· 1054 1054 dma_entry_free(entry); 1055 1055 } 1056 1056 1057 - static void check_for_stack(struct device *dev, 1058 - struct page *page, size_t offset) 1057 + static void check_for_stack(struct device *dev, phys_addr_t phys) 1059 1058 { 1060 1059 void *addr; 1061 1060 struct vm_struct *stack_vm_area = task_stack_vm_area(current); 1062 1061 1063 1062 if (!stack_vm_area) { 1064 1063 /* Stack is direct-mapped. */ 1065 - if (PageHighMem(page)) 1064 + if (PhysHighMem(phys)) 1066 1065 return; 1067 - addr = page_address(page) + offset; 1066 + addr = phys_to_virt(phys); 1068 1067 if (object_is_on_stack(addr)) 1069 1068 err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr); 1070 1069 } else { ··· 1071 1072 int i; 1072 1073 1073 1074 for (i = 0; i < stack_vm_area->nr_pages; i++) { 1074 - if (page != stack_vm_area->pages[i]) 1075 + if (__phys_to_pfn(phys) != 1076 + page_to_pfn(stack_vm_area->pages[i])) 1075 1077 continue; 1076 1078 1077 - addr = (u8 *)current->stack + i * PAGE_SIZE + offset; 1079 + addr = (u8 *)current->stack + i * PAGE_SIZE + 1080 + (phys % PAGE_SIZE); 1078 1081 err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr); 1079 1082 break; 1080 1083 } ··· 1205 1204 } 1206 1205 EXPORT_SYMBOL(debug_dma_map_single); 1207 1206 1208 - void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, 1209 - size_t size, int direction, dma_addr_t dma_addr, 1210 - unsigned long attrs) 1207 + void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 1208 + int direction, dma_addr_t dma_addr, unsigned long attrs) 1211 1209 { 1212 1210 struct dma_debug_entry *entry; 1213 1211 ··· 1221 1221 return; 1222 1222 1223 1223 entry->dev = dev; 1224 - entry->type = dma_debug_single; 1225 - entry->paddr = page_to_phys(page) + offset; 1224 + entry->type = dma_debug_phy; 1225 + entry->paddr = phys; 1226 1226 entry->dev_addr = dma_addr; 1227 1227 entry->size = size; 1228 1228 entry->direction = direction; 1229 1229 entry->map_err_type = MAP_ERR_NOT_CHECKED; 1230 1230 1231 - check_for_stack(dev, page, offset); 1231 + if (!(attrs & DMA_ATTR_MMIO)) { 1232 + check_for_stack(dev, phys); 1232 1233 1233 - if (!PageHighMem(page)) { 1234 - void *addr = page_address(page) + offset; 1235 - 1236 - check_for_illegal_area(dev, addr, size); 1234 + if (!PhysHighMem(phys)) 1235 + check_for_illegal_area(dev, phys_to_virt(phys), size); 1237 1236 } 1238 1237 1239 1238 add_dma_entry(entry, attrs); ··· 1276 1277 } 1277 1278 EXPORT_SYMBOL(debug_dma_mapping_error); 1278 1279 1279 - void debug_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, 1280 + void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, 1280 1281 size_t size, int direction) 1281 1282 { 1282 1283 struct dma_debug_entry ref = { 1283 - .type = dma_debug_single, 1284 + .type = dma_debug_phy, 1284 1285 .dev = dev, 1285 1286 .dev_addr = dma_addr, 1286 1287 .size = size, ··· 1304 1305 return; 1305 1306 1306 1307 for_each_sg(sg, s, nents, i) { 1307 - check_for_stack(dev, sg_page(s), s->offset); 1308 + check_for_stack(dev, sg_phys(s)); 1308 1309 if (!PageHighMem(sg_page(s))) 1309 1310 check_for_illegal_area(dev, sg_virt(s), s->length); 1310 1311 } ··· 1437 1438 return; 1438 1439 1439 1440 ref.paddr = virt_to_paddr(virt); 1440 - 1441 - if (unlikely(dma_debug_disabled())) 1442 - return; 1443 - 1444 - check_unmap(&ref); 1445 - } 1446 - 1447 - void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size, 1448 - int direction, dma_addr_t dma_addr, 1449 - unsigned long attrs) 1450 - { 1451 - struct dma_debug_entry *entry; 1452 - 1453 - if (unlikely(dma_debug_disabled())) 1454 - return; 1455 - 1456 - entry = dma_entry_alloc(); 1457 - if (!entry) 1458 - return; 1459 - 1460 - entry->type = dma_debug_resource; 1461 - entry->dev = dev; 1462 - entry->paddr = addr; 1463 - entry->size = size; 1464 - entry->dev_addr = dma_addr; 1465 - entry->direction = direction; 1466 - entry->map_err_type = MAP_ERR_NOT_CHECKED; 1467 - 1468 - add_dma_entry(entry, attrs); 1469 - } 1470 - 1471 - void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr, 1472 - size_t size, int direction) 1473 - { 1474 - struct dma_debug_entry ref = { 1475 - .type = dma_debug_resource, 1476 - .dev = dev, 1477 - .dev_addr = dma_addr, 1478 - .size = size, 1479 - .direction = direction, 1480 - }; 1481 1441 1482 1442 if (unlikely(dma_debug_disabled())) 1483 1443 return;
+7 -30
kernel/dma/debug.h
··· 9 9 #define _KERNEL_DMA_DEBUG_H 10 10 11 11 #ifdef CONFIG_DMA_API_DEBUG 12 - extern void debug_dma_map_page(struct device *dev, struct page *page, 13 - size_t offset, size_t size, 14 - int direction, dma_addr_t dma_addr, 12 + extern void debug_dma_map_phys(struct device *dev, phys_addr_t phys, 13 + size_t size, int direction, dma_addr_t dma_addr, 15 14 unsigned long attrs); 16 15 17 - extern void debug_dma_unmap_page(struct device *dev, dma_addr_t addr, 16 + extern void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, 18 17 size_t size, int direction); 19 18 20 19 extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, ··· 29 30 30 31 extern void debug_dma_free_coherent(struct device *dev, size_t size, 31 32 void *virt, dma_addr_t addr); 32 - 33 - extern void debug_dma_map_resource(struct device *dev, phys_addr_t addr, 34 - size_t size, int direction, 35 - dma_addr_t dma_addr, 36 - unsigned long attrs); 37 - 38 - extern void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr, 39 - size_t size, int direction); 40 33 41 34 extern void debug_dma_sync_single_for_cpu(struct device *dev, 42 35 dma_addr_t dma_handle, size_t size, ··· 53 62 size_t size, int direction, 54 63 dma_addr_t dma_addr); 55 64 #else /* CONFIG_DMA_API_DEBUG */ 56 - static inline void debug_dma_map_page(struct device *dev, struct page *page, 57 - size_t offset, size_t size, 58 - int direction, dma_addr_t dma_addr, 59 - unsigned long attrs) 65 + static inline void debug_dma_map_phys(struct device *dev, phys_addr_t phys, 66 + size_t size, int direction, 67 + dma_addr_t dma_addr, unsigned long attrs) 60 68 { 61 69 } 62 70 63 - static inline void debug_dma_unmap_page(struct device *dev, dma_addr_t addr, 71 + static inline void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, 64 72 size_t size, int direction) 65 73 { 66 74 } ··· 84 94 85 95 static inline void debug_dma_free_coherent(struct device *dev, size_t size, 86 96 void *virt, dma_addr_t addr) 87 - { 88 - } 89 - 90 - static inline void debug_dma_map_resource(struct device *dev, phys_addr_t addr, 91 - size_t size, int direction, 92 - dma_addr_t dma_addr, 93 - unsigned long attrs) 94 - { 95 - } 96 - 97 - static inline void debug_dma_unmap_resource(struct device *dev, 98 - dma_addr_t dma_addr, size_t size, 99 - int direction) 100 97 { 101 98 } 102 99
+16 -37
kernel/dma/direct.c
··· 120 120 gfp_t gfp, bool allow_highmem) 121 121 { 122 122 int node = dev_to_node(dev); 123 - struct page *page = NULL; 123 + struct page *page; 124 124 u64 phys_limit; 125 125 126 126 WARN_ON_ONCE(!PAGE_ALIGNED(size)); ··· 131 131 gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); 132 132 page = dma_alloc_contiguous(dev, size, gfp); 133 133 if (page) { 134 - if (!dma_coherent_ok(dev, page_to_phys(page), size) || 135 - (!allow_highmem && PageHighMem(page))) { 136 - dma_free_contiguous(dev, page, size); 137 - page = NULL; 138 - } 134 + if (dma_coherent_ok(dev, page_to_phys(page), size) && 135 + (allow_highmem || !PageHighMem(page))) 136 + return page; 137 + 138 + dma_free_contiguous(dev, page, size); 139 139 } 140 - again: 141 - if (!page) 142 - page = alloc_pages_node(node, gfp, get_order(size)); 143 - if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { 140 + 141 + while ((page = alloc_pages_node(node, gfp, get_order(size))) 142 + && !dma_coherent_ok(dev, page_to_phys(page), size)) { 144 143 __free_pages(page, get_order(size)); 145 - page = NULL; 146 144 147 145 if (IS_ENABLED(CONFIG_ZONE_DMA32) && 148 146 phys_limit < DMA_BIT_MASK(64) && 149 - !(gfp & (GFP_DMA32 | GFP_DMA))) { 147 + !(gfp & (GFP_DMA32 | GFP_DMA))) 150 148 gfp |= GFP_DMA32; 151 - goto again; 152 - } 153 - 154 - if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { 149 + else if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) 155 150 gfp = (gfp & ~GFP_DMA32) | GFP_DMA; 156 - goto again; 157 - } 151 + else 152 + return NULL; 158 153 } 159 154 160 155 return page; ··· 448 453 if (sg_dma_is_bus_address(sg)) 449 454 sg_dma_unmark_bus_address(sg); 450 455 else 451 - dma_direct_unmap_page(dev, sg->dma_address, 456 + dma_direct_unmap_phys(dev, sg->dma_address, 452 457 sg_dma_len(sg), dir, attrs); 453 458 } 454 459 } ··· 471 476 */ 472 477 break; 473 478 case PCI_P2PDMA_MAP_NONE: 474 - sg->dma_address = dma_direct_map_page(dev, sg_page(sg), 475 - sg->offset, sg->length, dir, attrs); 479 + sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg), 480 + sg->length, dir, attrs); 476 481 if (sg->dma_address == DMA_MAPPING_ERROR) { 477 482 ret = -EIO; 478 483 goto out_unmap; ··· 495 500 out_unmap: 496 501 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); 497 502 return ret; 498 - } 499 - 500 - dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, 501 - size_t size, enum dma_data_direction dir, unsigned long attrs) 502 - { 503 - dma_addr_t dma_addr = paddr; 504 - 505 - if (unlikely(!dma_capable(dev, dma_addr, size, false))) { 506 - dev_err_once(dev, 507 - "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 508 - &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 509 - WARN_ON_ONCE(1); 510 - return DMA_MAPPING_ERROR; 511 - } 512 - 513 - return dma_addr; 514 503 } 515 504 516 505 int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
+35 -20
kernel/dma/direct.h
··· 80 80 arch_dma_mark_clean(paddr, size); 81 81 } 82 82 83 - static inline dma_addr_t dma_direct_map_page(struct device *dev, 84 - struct page *page, unsigned long offset, size_t size, 85 - enum dma_data_direction dir, unsigned long attrs) 83 + static inline dma_addr_t dma_direct_map_phys(struct device *dev, 84 + phys_addr_t phys, size_t size, enum dma_data_direction dir, 85 + unsigned long attrs) 86 86 { 87 - phys_addr_t phys = page_to_phys(page) + offset; 88 - dma_addr_t dma_addr = phys_to_dma(dev, phys); 87 + dma_addr_t dma_addr; 89 88 90 89 if (is_swiotlb_force_bounce(dev)) { 91 - if (is_pci_p2pdma_page(page)) 92 - return DMA_MAPPING_ERROR; 90 + if (attrs & DMA_ATTR_MMIO) 91 + goto err_overflow; 92 + 93 93 return swiotlb_map(dev, phys, size, dir, attrs); 94 94 } 95 95 96 - if (unlikely(!dma_capable(dev, dma_addr, size, true)) || 97 - dma_kmalloc_needs_bounce(dev, size, dir)) { 98 - if (is_pci_p2pdma_page(page)) 99 - return DMA_MAPPING_ERROR; 100 - if (is_swiotlb_active(dev)) 101 - return swiotlb_map(dev, phys, size, dir, attrs); 96 + if (attrs & DMA_ATTR_MMIO) { 97 + dma_addr = phys; 98 + if (unlikely(!dma_capable(dev, dma_addr, size, false))) 99 + goto err_overflow; 100 + } else { 101 + dma_addr = phys_to_dma(dev, phys); 102 + if (unlikely(!dma_capable(dev, dma_addr, size, true)) || 103 + dma_kmalloc_needs_bounce(dev, size, dir)) { 104 + if (is_swiotlb_active(dev)) 105 + return swiotlb_map(dev, phys, size, dir, attrs); 102 106 103 - dev_WARN_ONCE(dev, 1, 104 - "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 105 - &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 106 - return DMA_MAPPING_ERROR; 107 + goto err_overflow; 108 + } 107 109 } 108 110 109 - if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 111 + if (!dev_is_dma_coherent(dev) && 112 + !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) 110 113 arch_sync_dma_for_device(phys, size, dir); 111 114 return dma_addr; 115 + 116 + err_overflow: 117 + dev_WARN_ONCE( 118 + dev, 1, 119 + "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 120 + &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 121 + return DMA_MAPPING_ERROR; 112 122 } 113 123 114 - static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, 124 + static inline void dma_direct_unmap_phys(struct device *dev, dma_addr_t addr, 115 125 size_t size, enum dma_data_direction dir, unsigned long attrs) 116 126 { 117 - phys_addr_t phys = dma_to_phys(dev, addr); 127 + phys_addr_t phys; 118 128 129 + if (attrs & DMA_ATTR_MMIO) 130 + /* nothing to do: uncached and no swiotlb */ 131 + return; 132 + 133 + phys = dma_to_phys(dev, addr); 119 134 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 120 135 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 121 136
+67 -45
kernel/dma/mapping.c
··· 152 152 return dma_go_direct(dev, *dev->dma_mask, ops); 153 153 } 154 154 155 - dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 156 - size_t offset, size_t size, enum dma_data_direction dir, 157 - unsigned long attrs) 155 + dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 156 + enum dma_data_direction dir, unsigned long attrs) 158 157 { 159 158 const struct dma_map_ops *ops = get_dma_ops(dev); 159 + bool is_mmio = attrs & DMA_ATTR_MMIO; 160 160 dma_addr_t addr; 161 161 162 162 BUG_ON(!valid_dma_direction(dir)); ··· 165 165 return DMA_MAPPING_ERROR; 166 166 167 167 if (dma_map_direct(dev, ops) || 168 - arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) 169 - addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 168 + (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) 169 + addr = dma_direct_map_phys(dev, phys, size, dir, attrs); 170 170 else if (use_dma_iommu(dev)) 171 - addr = iommu_dma_map_page(dev, page, offset, size, dir, attrs); 172 - else 171 + addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); 172 + else if (is_mmio) { 173 + if (!ops->map_resource) 174 + return DMA_MAPPING_ERROR; 175 + 176 + addr = ops->map_resource(dev, phys, size, dir, attrs); 177 + } else { 178 + struct page *page = phys_to_page(phys); 179 + size_t offset = offset_in_page(phys); 180 + 181 + /* 182 + * The dma_ops API contract for ops->map_page() requires 183 + * kmappable memory, while ops->map_resource() does not. 184 + */ 173 185 addr = ops->map_page(dev, page, offset, size, dir, attrs); 174 - kmsan_handle_dma(page, offset, size, dir); 175 - trace_dma_map_page(dev, page_to_phys(page) + offset, addr, size, dir, 176 - attrs); 177 - debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); 186 + } 187 + 188 + if (!is_mmio) 189 + kmsan_handle_dma(phys, size, dir); 190 + trace_dma_map_phys(dev, phys, addr, size, dir, attrs); 191 + debug_dma_map_phys(dev, phys, size, dir, addr, attrs); 178 192 179 193 return addr; 180 194 } 195 + EXPORT_SYMBOL_GPL(dma_map_phys); 196 + 197 + dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 198 + size_t offset, size_t size, enum dma_data_direction dir, 199 + unsigned long attrs) 200 + { 201 + phys_addr_t phys = page_to_phys(page) + offset; 202 + 203 + if (unlikely(attrs & DMA_ATTR_MMIO)) 204 + return DMA_MAPPING_ERROR; 205 + 206 + if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && 207 + WARN_ON_ONCE(is_zone_device_page(page))) 208 + return DMA_MAPPING_ERROR; 209 + 210 + return dma_map_phys(dev, phys, size, dir, attrs); 211 + } 181 212 EXPORT_SYMBOL(dma_map_page_attrs); 182 213 183 - void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 214 + void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, 184 215 enum dma_data_direction dir, unsigned long attrs) 185 216 { 186 217 const struct dma_map_ops *ops = get_dma_ops(dev); 218 + bool is_mmio = attrs & DMA_ATTR_MMIO; 187 219 188 220 BUG_ON(!valid_dma_direction(dir)); 189 221 if (dma_map_direct(dev, ops) || 190 - arch_dma_unmap_page_direct(dev, addr + size)) 191 - dma_direct_unmap_page(dev, addr, size, dir, attrs); 222 + (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) 223 + dma_direct_unmap_phys(dev, addr, size, dir, attrs); 192 224 else if (use_dma_iommu(dev)) 193 - iommu_dma_unmap_page(dev, addr, size, dir, attrs); 194 - else 225 + iommu_dma_unmap_phys(dev, addr, size, dir, attrs); 226 + else if (is_mmio) { 227 + if (ops->unmap_resource) 228 + ops->unmap_resource(dev, addr, size, dir, attrs); 229 + } else 195 230 ops->unmap_page(dev, addr, size, dir, attrs); 196 - trace_dma_unmap_page(dev, addr, size, dir, attrs); 197 - debug_dma_unmap_page(dev, addr, size, dir); 231 + trace_dma_unmap_phys(dev, addr, size, dir, attrs); 232 + debug_dma_unmap_phys(dev, addr, size, dir); 233 + } 234 + EXPORT_SYMBOL_GPL(dma_unmap_phys); 235 + 236 + void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 237 + enum dma_data_direction dir, unsigned long attrs) 238 + { 239 + if (unlikely(attrs & DMA_ATTR_MMIO)) 240 + return; 241 + 242 + dma_unmap_phys(dev, addr, size, dir, attrs); 198 243 } 199 244 EXPORT_SYMBOL(dma_unmap_page_attrs); 200 245 ··· 366 321 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 367 322 size_t size, enum dma_data_direction dir, unsigned long attrs) 368 323 { 369 - const struct dma_map_ops *ops = get_dma_ops(dev); 370 - dma_addr_t addr = DMA_MAPPING_ERROR; 371 - 372 - BUG_ON(!valid_dma_direction(dir)); 373 - 374 - if (WARN_ON_ONCE(!dev->dma_mask)) 324 + if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && 325 + WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) 375 326 return DMA_MAPPING_ERROR; 376 327 377 - if (dma_map_direct(dev, ops)) 378 - addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); 379 - else if (use_dma_iommu(dev)) 380 - addr = iommu_dma_map_resource(dev, phys_addr, size, dir, attrs); 381 - else if (ops->map_resource) 382 - addr = ops->map_resource(dev, phys_addr, size, dir, attrs); 383 - 384 - trace_dma_map_resource(dev, phys_addr, addr, size, dir, attrs); 385 - debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); 386 - return addr; 328 + return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); 387 329 } 388 330 EXPORT_SYMBOL(dma_map_resource); 389 331 390 332 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 391 333 enum dma_data_direction dir, unsigned long attrs) 392 334 { 393 - const struct dma_map_ops *ops = get_dma_ops(dev); 394 - 395 - BUG_ON(!valid_dma_direction(dir)); 396 - if (dma_map_direct(dev, ops)) 397 - ; /* nothing to do: uncached and no swiotlb */ 398 - else if (use_dma_iommu(dev)) 399 - iommu_dma_unmap_resource(dev, addr, size, dir, attrs); 400 - else if (ops->unmap_resource) 401 - ops->unmap_resource(dev, addr, size, dir, attrs); 402 - trace_dma_unmap_resource(dev, addr, size, dir, attrs); 403 - debug_dma_unmap_resource(dev, addr, size, dir); 335 + dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO); 404 336 } 405 337 EXPORT_SYMBOL(dma_unmap_resource); 406 338
+3 -3
kernel/dma/ops_helpers.c
··· 72 72 return NULL; 73 73 74 74 if (use_dma_iommu(dev)) 75 - *dma_handle = iommu_dma_map_page(dev, page, 0, size, dir, 76 - DMA_ATTR_SKIP_CPU_SYNC); 75 + *dma_handle = iommu_dma_map_phys(dev, page_to_phys(page), size, 76 + dir, DMA_ATTR_SKIP_CPU_SYNC); 77 77 else 78 78 *dma_handle = ops->map_page(dev, page, 0, size, dir, 79 79 DMA_ATTR_SKIP_CPU_SYNC); ··· 92 92 const struct dma_map_ops *ops = get_dma_ops(dev); 93 93 94 94 if (use_dma_iommu(dev)) 95 - iommu_dma_unmap_page(dev, dma_handle, size, dir, 95 + iommu_dma_unmap_phys(dev, dma_handle, size, dir, 96 96 DMA_ATTR_SKIP_CPU_SYNC); 97 97 else if (ops->unmap_page) 98 98 ops->unmap_page(dev, dma_handle, size, dir,
+1 -1
kernel/dma/swiotlb.c
··· 1209 1209 nslabs = nr_slots(alloc_size); 1210 1210 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit); 1211 1211 pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit, 1212 - GFP_NOWAIT | __GFP_NOWARN); 1212 + GFP_NOWAIT); 1213 1213 if (!pool) 1214 1214 return -1; 1215 1215
+10 -9
mm/hmm.c
··· 806 806 case PCI_P2PDMA_MAP_NONE: 807 807 break; 808 808 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 809 - attrs |= DMA_ATTR_SKIP_CPU_SYNC; 809 + attrs |= DMA_ATTR_MMIO; 810 810 pfns[idx] |= HMM_PFN_P2PDMA; 811 811 break; 812 812 case PCI_P2PDMA_MAP_BUS_ADDR: ··· 835 835 if (WARN_ON_ONCE(dma_need_unmap(dev) && !dma_addrs)) 836 836 goto error; 837 837 838 - dma_addr = dma_map_page(dev, page, 0, map->dma_entry_size, 839 - DMA_BIDIRECTIONAL); 838 + dma_addr = dma_map_phys(dev, paddr, map->dma_entry_size, 839 + DMA_BIDIRECTIONAL, attrs); 840 840 if (dma_mapping_error(dev, dma_addr)) 841 841 goto error; 842 842 ··· 871 871 if ((pfns[idx] & valid_dma) != valid_dma) 872 872 return false; 873 873 874 + if (pfns[idx] & HMM_PFN_P2PDMA) 875 + attrs |= DMA_ATTR_MMIO; 876 + 874 877 if (pfns[idx] & HMM_PFN_P2PDMA_BUS) 875 878 ; /* no need to unmap bus address P2P mappings */ 876 - else if (dma_use_iova(state)) { 877 - if (pfns[idx] & HMM_PFN_P2PDMA) 878 - attrs |= DMA_ATTR_SKIP_CPU_SYNC; 879 + else if (dma_use_iova(state)) 879 880 dma_iova_unlink(dev, state, idx * map->dma_entry_size, 880 881 map->dma_entry_size, DMA_BIDIRECTIONAL, attrs); 881 - } else if (dma_need_unmap(dev)) 882 - dma_unmap_page(dev, dma_addrs[idx], map->dma_entry_size, 883 - DMA_BIDIRECTIONAL); 882 + else if (dma_need_unmap(dev)) 883 + dma_unmap_phys(dev, dma_addrs[idx], map->dma_entry_size, 884 + DMA_BIDIRECTIONAL, attrs); 884 885 885 886 pfns[idx] &= 886 887 ~(HMM_PFN_DMA_MAPPED | HMM_PFN_P2PDMA | HMM_PFN_P2PDMA_BUS);
+7 -6
mm/kmsan/hooks.c
··· 336 336 } 337 337 338 338 /* Helper function to handle DMA data transfers. */ 339 - void kmsan_handle_dma(struct page *page, size_t offset, size_t size, 339 + void kmsan_handle_dma(phys_addr_t phys, size_t size, 340 340 enum dma_data_direction dir) 341 341 { 342 - u64 page_offset, to_go, addr; 342 + struct page *page = phys_to_page(phys); 343 + u64 page_offset, to_go; 344 + void *addr; 343 345 344 - if (PageHighMem(page)) 346 + if (PhysHighMem(phys)) 345 347 return; 346 - addr = (u64)page_address(page) + offset; 348 + addr = page_to_virt(page); 347 349 /* 348 350 * The kernel may occasionally give us adjacent DMA pages not belonging 349 351 * to the same allocation. Process them separately to avoid triggering ··· 368 366 int i; 369 367 370 368 for_each_sg(sg, item, nents, i) 371 - kmsan_handle_dma(sg_page(item), item->offset, item->length, 372 - dir); 369 + kmsan_handle_dma(sg_phys(item), item->length, dir); 373 370 } 374 371 375 372 /* Functions from kmsan-checks.h follow. */
+3
rust/kernel/dma.rs
··· 252 252 /// Indicates that the buffer is fully accessible at an elevated privilege level (and 253 253 /// ideally inaccessible or at least read-only at lesser-privileged levels). 254 254 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); 255 + 256 + /// Indicates that the buffer is MMIO memory. 257 + pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO); 255 258 } 256 259 257 260 /// DMA data direction.
+1 -1
tools/virtio/linux/kmsan.h
··· 4 4 5 5 #include <linux/gfp.h> 6 6 7 - inline void kmsan_handle_dma(struct page *page, size_t offset, size_t size, 7 + inline void kmsan_handle_dma(phys_addr_t phys, size_t size, 8 8 enum dma_data_direction dir) 9 9 { 10 10 }