dma-mapping: Support batch mode for dma_direct_{map,unmap}_sg

Extending these APIs with a flush argument:
dma_direct_unmap_phys(), dma_direct_map_phys(), and
dma_direct_sync_single_for_cpu(). For single-buffer cases, flush=true
would be used, while for SG cases flush=false would be used, followed
by a single flush after all cache operations are issued in
dma_direct_{map,unmap}_sg().

This ultimately benefits dma_map_sg() and dma_unmap_sg().

Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tangquan Zheng <zhengtangquan@oppo.com>
Reviewed-by: Leon Romanovsky <leon@kernel.org>
Tested-by: Xueyuan Chen <xueyuan.chen21@gmail.com>
Signed-off-by: Barry Song <baohua@kernel.org>
Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
Link: https://lore.kernel.org/r/20260228221337.59951-1-21cnbao@gmail.com

authored by

Barry Song and committed by

Marek Szyprowski 2 months ago 661f8a19 d7eafe65

+26 -13

3 changed files

expand all

kernel

dma

direct.c

direct.h

mapping.c

+13 -4

kernel/dma/direct.c

··· 444 444 { 445 445 struct scatterlist *sg; 446 446 int i; 447 + bool need_sync = false; 447 448 448 449 for_each_sg(sgl, sg, nents, i) { 449 - if (sg_dma_is_bus_address(sg)) 450 + if (sg_dma_is_bus_address(sg)) { 450 451 sg_dma_unmark_bus_address(sg); 451 - else 452 + } else { 453 + need_sync = true; 452 454 dma_direct_unmap_phys(dev, sg->dma_address, 453 - sg_dma_len(sg), dir, attrs); 455 + sg_dma_len(sg), dir, attrs, false); 456 + } 454 457 } 458 + if (need_sync && !dev_is_dma_coherent(dev)) 459 + arch_sync_dma_flush(); 455 460 } 456 461 #endif 457 462 ··· 466 461 struct pci_p2pdma_map_state p2pdma_state = {}; 467 462 struct scatterlist *sg; 468 463 int i, ret; 464 + bool need_sync = false; 469 465 470 466 for_each_sg(sgl, sg, nents, i) { 471 467 switch (pci_p2pdma_state(&p2pdma_state, dev, sg_page(sg))) { ··· 478 472 */ 479 473 break; 480 474 case PCI_P2PDMA_MAP_NONE: 475 + need_sync = true; 481 476 sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg), 482 - sg->length, dir, attrs); 477 + sg->length, dir, attrs, false); 483 478 if (sg->dma_address == DMA_MAPPING_ERROR) { 484 479 ret = -EIO; 485 480 goto out_unmap; ··· 499 492 sg_dma_len(sg) = sg->length; 500 493 } 501 494 495 + if (need_sync && !dev_is_dma_coherent(dev)) 496 + arch_sync_dma_flush(); 502 497 return nents; 503 498 504 499 out_unmap:

+10 -6

kernel/dma/direct.h

··· 67 67 } 68 68 69 69 static inline void dma_direct_sync_single_for_cpu(struct device *dev, 70 - dma_addr_t addr, size_t size, enum dma_data_direction dir) 70 + dma_addr_t addr, size_t size, enum dma_data_direction dir, 71 + bool flush) 71 72 { 72 73 phys_addr_t paddr = dma_to_phys(dev, addr); 73 74 74 75 if (!dev_is_dma_coherent(dev)) { 75 76 arch_sync_dma_for_cpu(paddr, size, dir); 76 - arch_sync_dma_flush(); 77 + if (flush) 78 + arch_sync_dma_flush(); 77 79 arch_sync_dma_for_cpu_all(); 78 80 } 79 81 ··· 84 82 85 83 static inline dma_addr_t dma_direct_map_phys(struct device *dev, 86 84 phys_addr_t phys, size_t size, enum dma_data_direction dir, 87 - unsigned long attrs) 85 + unsigned long attrs, bool flush) 88 86 { 89 87 dma_addr_t dma_addr; 90 88 ··· 113 111 if (!dev_is_dma_coherent(dev) && 114 112 !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) { 115 113 arch_sync_dma_for_device(phys, size, dir); 116 - arch_sync_dma_flush(); 114 + if (flush) 115 + arch_sync_dma_flush(); 117 116 } 118 117 return dma_addr; 119 118 ··· 127 124 } 128 125 129 126 static inline void dma_direct_unmap_phys(struct device *dev, dma_addr_t addr, 130 - size_t size, enum dma_data_direction dir, unsigned long attrs) 127 + size_t size, enum dma_data_direction dir, unsigned long attrs, 128 + bool flush) 131 129 { 132 130 phys_addr_t phys; 133 131 ··· 138 134 139 135 phys = dma_to_phys(dev, addr); 140 136 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 141 - dma_direct_sync_single_for_cpu(dev, addr, size, dir); 137 + dma_direct_sync_single_for_cpu(dev, addr, size, dir, flush); 142 138 143 139 swiotlb_tbl_unmap_single(dev, phys, size, dir, 144 140 attrs | DMA_ATTR_SKIP_CPU_SYNC);

+3 -3

kernel/dma/mapping.c

··· 166 166 167 167 if (dma_map_direct(dev, ops) || 168 168 (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) 169 - addr = dma_direct_map_phys(dev, phys, size, dir, attrs); 169 + addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true); 170 170 else if (use_dma_iommu(dev)) 171 171 addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); 172 172 else if (ops->map_phys) ··· 207 207 BUG_ON(!valid_dma_direction(dir)); 208 208 if (dma_map_direct(dev, ops) || 209 209 (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) 210 - dma_direct_unmap_phys(dev, addr, size, dir, attrs); 210 + dma_direct_unmap_phys(dev, addr, size, dir, attrs, true); 211 211 else if (use_dma_iommu(dev)) 212 212 iommu_dma_unmap_phys(dev, addr, size, dir, attrs); 213 213 else if (ops->unmap_phys) ··· 373 373 374 374 BUG_ON(!valid_dma_direction(dir)); 375 375 if (dma_map_direct(dev, ops)) 376 - dma_direct_sync_single_for_cpu(dev, addr, size, dir); 376 + dma_direct_sync_single_for_cpu(dev, addr, size, dir, true); 377 377 else if (use_dma_iommu(dev)) 378 378 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); 379 379 else if (ops->sync_single_for_cpu)

Configure Feed

Configure Feed