Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-7.0-2026-03-25' into dma-mapping-for-next
dma-mapping fixes for Linux 7.0
A set of fixes for DMA-mapping subsystem, which resolve false-positive
warnings from KMSAN and DMA-API debug (Shigeru Yoshida and Leon
Romanovsky) as well as a simple build fix (Miguel Ojeda).
Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
+111
-37
+29
-7
Documentation/core-api/dma-attributes.rst
+29
-7
Documentation/core-api/dma-attributes.rst
···
149
149
DMA_ATTR_MMIO will not perform any cache flushing. The address
150
150
provided must never be mapped cacheable into the CPU.
151
151
152
-
DMA_ATTR_CPU_CACHE_CLEAN
153
-
------------------------
152
+
DMA_ATTR_DEBUGGING_IGNORE_CACHELINES
153
+
------------------------------------
154
154
155
-
This attribute indicates the CPU will not dirty any cacheline overlapping this
156
-
DMA_FROM_DEVICE/DMA_BIDIRECTIONAL buffer while it is mapped. This allows
157
-
multiple small buffers to safely share a cacheline without risk of data
158
-
corruption, suppressing DMA debug warnings about overlapping mappings.
159
-
All mappings sharing a cacheline should have this attribute.
155
+
This attribute indicates that CPU cache lines may overlap for buffers mapped
156
+
with DMA_FROM_DEVICE or DMA_BIDIRECTIONAL.
157
+
158
+
Such overlap may occur when callers map multiple small buffers that reside
159
+
within the same cache line. In this case, callers must guarantee that the CPU
160
+
will not dirty these cache lines after the mappings are established. When this
161
+
condition is met, multiple buffers can safely share a cache line without risking
162
+
data corruption.
163
+
164
+
All mappings that share a cache line must set this attribute to suppress DMA
165
+
debug warnings about overlapping mappings.
166
+
167
+
DMA_ATTR_REQUIRE_COHERENT
168
+
-------------------------
169
+
170
+
DMA mapping requests with the DMA_ATTR_REQUIRE_COHERENT fail on any
171
+
system where SWIOTLB or cache management is required. This should only
172
+
be used to support uAPI designs that require continuous HW DMA
173
+
coherence with userspace processes, for example RDMA and DRM. At a
174
+
minimum the memory being mapped must be userspace memory from
175
+
pin_user_pages() or similar.
176
+
177
+
Drivers should consider using dma_mmap_pages() instead of this
178
+
interface when building their uAPIs, when possible.
179
+
180
+
It must never be used in an in-kernel driver that only works with
181
+
kernel memory.
+2
arch/sparc/kernel/iommu.c
+2
arch/sparc/kernel/iommu.c
+2
arch/sparc/kernel/pci_sun4v.c
+2
arch/sparc/kernel/pci_sun4v.c
+3
-2
drivers/infiniband/core/umem.c
+3
-2
drivers/infiniband/core/umem.c
···
55
55
56
56
if (dirty)
57
57
ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
58
-
DMA_BIDIRECTIONAL, 0);
58
+
DMA_BIDIRECTIONAL,
59
+
DMA_ATTR_REQUIRE_COHERENT);
59
60
60
61
for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) {
61
62
unpin_user_page_range_dirty_lock(sg_page(sg),
···
170
169
unsigned long lock_limit;
171
170
unsigned long new_pinned;
172
171
unsigned long cur_base;
173
-
unsigned long dma_attr = 0;
172
+
unsigned long dma_attr = DMA_ATTR_REQUIRE_COHERENT;
174
173
struct mm_struct *mm;
175
174
unsigned long npages;
176
175
int pinned, ret;
+17
-4
drivers/iommu/dma-iommu.c
+17
-4
drivers/iommu/dma-iommu.c
···
1219
1219
*/
1220
1220
if (dev_use_swiotlb(dev, size, dir) &&
1221
1221
iova_unaligned(iovad, phys, size)) {
1222
-
if (attrs & DMA_ATTR_MMIO)
1222
+
if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
1223
1223
return DMA_MAPPING_ERROR;
1224
1224
1225
1225
phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs);
···
1233
1233
}
1234
1234
1235
1235
iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
1236
-
if (iova == DMA_MAPPING_ERROR && !(attrs & DMA_ATTR_MMIO))
1236
+
if (iova == DMA_MAPPING_ERROR &&
1237
+
!(attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)))
1237
1238
swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1238
1239
return iova;
1239
1240
}
···
1244
1243
{
1245
1244
phys_addr_t phys;
1246
1245
1247
-
if (attrs & DMA_ATTR_MMIO) {
1246
+
if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) {
1248
1247
__iommu_dma_unmap(dev, dma_handle, size);
1249
1248
return;
1250
1249
}
···
1958
1957
if (WARN_ON_ONCE(iova_start_pad && offset > 0))
1959
1958
return -EIO;
1960
1959
1960
+
/*
1961
+
* DMA_IOVA_USE_SWIOTLB is set on state after some entry
1962
+
* took SWIOTLB path, which we were supposed to prevent
1963
+
* for DMA_ATTR_REQUIRE_COHERENT attribute.
1964
+
*/
1965
+
if (WARN_ON_ONCE((state->__size & DMA_IOVA_USE_SWIOTLB) &&
1966
+
(attrs & DMA_ATTR_REQUIRE_COHERENT)))
1967
+
return -EOPNOTSUPP;
1968
+
1969
+
if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
1970
+
return -EOPNOTSUPP;
1971
+
1961
1972
if (dev_use_swiotlb(dev, size, dir) &&
1962
1973
iova_unaligned(iovad, phys, size)) {
1963
-
if (attrs & DMA_ATTR_MMIO)
1974
+
if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
1964
1975
return -EPERM;
1965
1976
1966
1977
return iommu_dma_iova_link_swiotlb(dev, state, phys, offset,
+5
-5
drivers/virtio/virtio_ring.c
+5
-5
drivers/virtio/virtio_ring.c
···
2912
2912
* @data: the token identifying the buffer.
2913
2913
* @gfp: how to do memory allocations (if necessary).
2914
2914
*
2915
-
* Same as virtqueue_add_inbuf but passes DMA_ATTR_CPU_CACHE_CLEAN to indicate
2916
-
* that the CPU will not dirty any cacheline overlapping this buffer while it
2917
-
* is available, and to suppress overlapping cacheline warnings in DMA debug
2918
-
* builds.
2915
+
* Same as virtqueue_add_inbuf but passes DMA_ATTR_DEBUGGING_IGNORE_CACHELINES
2916
+
* to indicate that the CPU will not dirty any cacheline overlapping this buffer
2917
+
* while it is available, and to suppress overlapping cacheline warnings in DMA
2918
+
* debug builds.
2919
2919
*
2920
2920
* Caller must ensure we don't call this with other virtqueue operations
2921
2921
* at the same time (except where noted).
···
2928
2928
gfp_t gfp)
2929
2929
{
2930
2930
return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, false, gfp,
2931
-
DMA_ATTR_CPU_CACHE_CLEAN);
2931
+
DMA_ATTR_DEBUGGING_IGNORE_CACHELINES);
2932
2932
}
2933
2933
EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_cache_clean);
2934
2934
+13
-6
include/linux/dma-mapping.h
+13
-6
include/linux/dma-mapping.h
···
80
80
#define DMA_ATTR_MMIO (1UL << 10)
81
81
82
82
/*
83
-
* DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline
84
-
* overlapping this buffer while it is mapped for DMA. All mappings sharing
85
-
* a cacheline must have this attribute for this to be considered safe.
83
+
* DMA_ATTR_DEBUGGING_IGNORE_CACHELINES: Indicates the CPU cache line can be
84
+
* overlapped. All mappings sharing a cacheline must have this attribute for
85
+
* this to be considered safe.
86
86
*/
87
-
#define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11)
87
+
#define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES (1UL << 11)
88
+
89
+
/*
90
+
* DMA_ATTR_REQUIRE_COHERENT: Indicates that DMA coherency is required.
91
+
* All mappings that carry this attribute can't work with SWIOTLB and cache
92
+
* flushing.
93
+
*/
94
+
#define DMA_ATTR_REQUIRE_COHERENT (1UL << 12)
88
95
89
96
/*
90
97
* A dma_addr_t can hold any valid DMA or bus address for the platform. It can
···
255
248
{
256
249
return NULL;
257
250
}
258
-
static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
259
-
dma_addr_t dma_handle, unsigned long attrs)
251
+
static inline void dma_free_attrs(struct device *dev, size_t size,
252
+
void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
260
253
{
261
254
}
262
255
static inline void *dmam_alloc_attrs(struct device *dev, size_t size,
+3
-1
include/trace/events/dma.h
+3
-1
include/trace/events/dma.h
···
32
32
{ DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \
33
33
{ DMA_ATTR_NO_WARN, "NO_WARN" }, \
34
34
{ DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \
35
-
{ DMA_ATTR_MMIO, "MMIO" })
35
+
{ DMA_ATTR_MMIO, "MMIO" }, \
36
+
{ DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \
37
+
{ DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" })
36
38
37
39
DECLARE_EVENT_CLASS(dma_map,
38
40
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
+5
-4
kernel/dma/debug.c
+5
-4
kernel/dma/debug.c
···
453
453
return overlap;
454
454
}
455
455
456
-
static void active_cacheline_inc_overlap(phys_addr_t cln)
456
+
static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean)
457
457
{
458
458
int overlap = active_cacheline_read_overlap(cln);
459
459
···
462
462
/* If we overflowed the overlap counter then we're potentially
463
463
* leaking dma-mappings.
464
464
*/
465
-
WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
465
+
WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
466
466
pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
467
467
ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
468
468
}
···
495
495
if (rc == -EEXIST) {
496
496
struct dma_debug_entry *existing;
497
497
498
-
active_cacheline_inc_overlap(cln);
498
+
active_cacheline_inc_overlap(cln, entry->is_cache_clean);
499
499
existing = radix_tree_lookup(&dma_active_cacheline, cln);
500
500
/* A lookup failure here after we got -EEXIST is unexpected. */
501
501
WARN_ON(!existing);
···
601
601
unsigned long flags;
602
602
int rc;
603
603
604
-
entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN);
604
+
entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
605
+
DMA_ATTR_REQUIRE_COHERENT);
605
606
606
607
bucket = get_hash_bucket(entry, &flags);
607
608
hash_bucket_add(bucket, entry);
+5
-4
kernel/dma/direct.h
+5
-4
kernel/dma/direct.h
···
89
89
dma_addr_t dma_addr;
90
90
91
91
if (is_swiotlb_force_bounce(dev)) {
92
-
if (attrs & DMA_ATTR_MMIO)
93
-
goto err_overflow;
92
+
if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
93
+
return DMA_MAPPING_ERROR;
94
94
95
95
return swiotlb_map(dev, phys, size, dir, attrs);
96
96
}
···
103
103
dma_addr = phys_to_dma(dev, phys);
104
104
if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||
105
105
dma_kmalloc_needs_bounce(dev, size, dir)) {
106
-
if (is_swiotlb_active(dev))
106
+
if (is_swiotlb_active(dev) &&
107
+
!(attrs & DMA_ATTR_REQUIRE_COHERENT))
107
108
return swiotlb_map(dev, phys, size, dir, attrs);
108
109
109
110
goto err_overflow;
···
133
132
{
134
133
phys_addr_t phys;
135
134
136
-
if (attrs & DMA_ATTR_MMIO)
135
+
if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
137
136
/* nothing to do: uncached and no swiotlb */
138
137
return;
139
138
+6
kernel/dma/mapping.c
+6
kernel/dma/mapping.c
···
164
164
if (WARN_ON_ONCE(!dev->dma_mask))
165
165
return DMA_MAPPING_ERROR;
166
166
167
+
if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
168
+
return DMA_MAPPING_ERROR;
169
+
167
170
if (dma_map_direct(dev, ops) ||
168
171
(!is_mmio && arch_dma_map_phys_direct(dev, phys + size)))
169
172
addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true);
···
237
234
int ents;
238
235
239
236
BUG_ON(!valid_dma_direction(dir));
237
+
238
+
if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
239
+
return -EOPNOTSUPP;
240
240
241
241
if (WARN_ON_ONCE(!dev->dma_mask))
242
242
return 0;
+19
-2
kernel/dma/swiotlb.c
+19
-2
kernel/dma/swiotlb.c
···
30
30
#include <linux/gfp.h>
31
31
#include <linux/highmem.h>
32
32
#include <linux/io.h>
33
+
#include <linux/kmsan-checks.h>
33
34
#include <linux/iommu-helper.h>
34
35
#include <linux/init.h>
35
36
#include <linux/memblock.h>
···
905
904
906
905
local_irq_save(flags);
907
906
page = pfn_to_page(pfn);
908
-
if (dir == DMA_TO_DEVICE)
907
+
if (dir == DMA_TO_DEVICE) {
908
+
/*
909
+
* Ideally, kmsan_check_highmem_page()
910
+
* could be used here to detect infoleaks,
911
+
* but callers may map uninitialized buffers
912
+
* that will be written by the device,
913
+
* causing false positives.
914
+
*/
909
915
memcpy_from_page(vaddr, page, offset, sz);
910
-
else
916
+
} else {
917
+
kmsan_unpoison_memory(vaddr, sz);
911
918
memcpy_to_page(page, offset, vaddr, sz);
919
+
}
912
920
local_irq_restore(flags);
913
921
914
922
size -= sz;
···
926
916
offset = 0;
927
917
}
928
918
} else if (dir == DMA_TO_DEVICE) {
919
+
/*
920
+
* Ideally, kmsan_check_memory() could be used here to detect
921
+
* infoleaks (uninitialized data being sent to device), but
922
+
* callers may map uninitialized buffers that will be written
923
+
* by the device, causing false positives.
924
+
*/
929
925
memcpy(vaddr, phys_to_virt(orig_addr), size);
930
926
} else {
927
+
kmsan_unpoison_memory(vaddr, size);
931
928
memcpy(phys_to_virt(orig_addr), vaddr, size);
932
929
}
933
930
}
+2
-2
mm/hmm.c
+2
-2
mm/hmm.c
···
778
778
struct page *page = hmm_pfn_to_page(pfns[idx]);
779
779
phys_addr_t paddr = hmm_pfn_to_phys(pfns[idx]);
780
780
size_t offset = idx * map->dma_entry_size;
781
-
unsigned long attrs = 0;
781
+
unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT;
782
782
dma_addr_t dma_addr;
783
783
int ret;
784
784
···
871
871
struct dma_iova_state *state = &map->state;
872
872
dma_addr_t *dma_addrs = map->dma_list;
873
873
unsigned long *pfns = map->pfn_list;
874
-
unsigned long attrs = 0;
874
+
unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT;
875
875
876
876
if ((pfns[idx] & valid_dma) != valid_dma)
877
877
return false;