Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-5.7' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping updates from Christoph Hellwig:
- fix an integer overflow in the coherent pool (Kevin Grandemange)
- provide support for in-place uncached remapping and use that for
openrisc
- fix the arm coherent allocator to take the bus limit into account
* tag 'dma-mapping-5.7' of git://git.infradead.org/users/hch/dma-mapping:
ARM/dma-mapping: merge __dma_supported into arm_dma_supported
ARM/dma-mapping: take the bus limit into account in __dma_alloc
ARM/dma-mapping: remove get_coherent_dma_mask
openrisc: use the generic in-place uncached DMA allocator
dma-direct: provide a arch_dma_clear_uncached hook
dma-direct: make uncached_kernel_address more general
dma-direct: consolidate the error handling in dma_direct_alloc_pages
dma-direct: remove the cached_kernel_address hook
dma-coherent: fix integer overflow in the reserved-memory dma allocation
+76
-166
+11
-4
arch/Kconfig
+11
-4
arch/Kconfig
···
248
248
bool
249
249
250
250
#
251
-
# Select if arch has an uncached kernel segment and provides the
252
-
# uncached_kernel_address / cached_kernel_address symbols to use it
251
+
# Select if the architecture provides the arch_dma_set_uncached symbol to
252
+
# either provide an uncached segement alias for a DMA allocation, or
253
+
# to remap the page tables in place.
253
254
#
254
-
config ARCH_HAS_UNCACHED_SEGMENT
255
-
select ARCH_HAS_DMA_PREP_COHERENT
255
+
config ARCH_HAS_DMA_SET_UNCACHED
256
+
bool
257
+
258
+
#
259
+
# Select if the architectures provides the arch_dma_clear_uncached symbol
260
+
# to undo an in-place page table remap for uncached access.
261
+
#
262
+
config ARCH_HAS_DMA_CLEAR_UNCACHED
256
263
bool
257
264
258
265
# Select if arch init_task must go in the __init_task_data section
-2
arch/arm/include/asm/dma-iommu.h
-2
arch/arm/include/asm/dma-iommu.h
+18
-58
arch/arm/mm/dma-mapping.c
+18
-58
arch/arm/mm/dma-mapping.c
···
179
179
__dma_page_cpu_to_dev(page, offset, size, dir);
180
180
}
181
181
182
+
/*
183
+
* Return whether the given device DMA address mask can be supported
184
+
* properly. For example, if your device can only drive the low 24-bits
185
+
* during bus mastering, then you would pass 0x00ffffff as the mask
186
+
* to this function.
187
+
*/
188
+
static int arm_dma_supported(struct device *dev, u64 mask)
189
+
{
190
+
unsigned long max_dma_pfn = min(max_pfn - 1, arm_dma_pfn_limit);
191
+
192
+
/*
193
+
* Translate the device's DMA mask to a PFN limit. This
194
+
* PFN number includes the page which we can DMA to.
195
+
*/
196
+
return dma_to_pfn(dev, mask) >= max_dma_pfn;
197
+
}
198
+
182
199
const struct dma_map_ops arm_dma_ops = {
183
200
.alloc = arm_dma_alloc,
184
201
.free = arm_dma_free,
···
235
218
.get_required_mask = dma_direct_get_required_mask,
236
219
};
237
220
EXPORT_SYMBOL(arm_coherent_dma_ops);
238
-
239
-
static int __dma_supported(struct device *dev, u64 mask, bool warn)
240
-
{
241
-
unsigned long max_dma_pfn = min(max_pfn - 1, arm_dma_pfn_limit);
242
-
243
-
/*
244
-
* Translate the device's DMA mask to a PFN limit. This
245
-
* PFN number includes the page which we can DMA to.
246
-
*/
247
-
if (dma_to_pfn(dev, mask) < max_dma_pfn) {
248
-
if (warn)
249
-
dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
250
-
mask,
251
-
dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
252
-
max_dma_pfn + 1);
253
-
return 0;
254
-
}
255
-
256
-
return 1;
257
-
}
258
-
259
-
static u64 get_coherent_dma_mask(struct device *dev)
260
-
{
261
-
u64 mask = (u64)DMA_BIT_MASK(32);
262
-
263
-
if (dev) {
264
-
mask = dev->coherent_dma_mask;
265
-
266
-
/*
267
-
* Sanity check the DMA mask - it must be non-zero, and
268
-
* must be able to be satisfied by a DMA allocation.
269
-
*/
270
-
if (mask == 0) {
271
-
dev_warn(dev, "coherent DMA mask is unset\n");
272
-
return 0;
273
-
}
274
-
275
-
if (!__dma_supported(dev, mask, true))
276
-
return 0;
277
-
}
278
-
279
-
return mask;
280
-
}
281
221
282
222
static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag)
283
223
{
···
662
688
gfp_t gfp, pgprot_t prot, bool is_coherent,
663
689
unsigned long attrs, const void *caller)
664
690
{
665
-
u64 mask = get_coherent_dma_mask(dev);
691
+
u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
666
692
struct page *page = NULL;
667
693
void *addr;
668
694
bool allowblock, cma;
···
685
711
return NULL;
686
712
}
687
713
#endif
688
-
689
-
if (!mask)
690
-
return NULL;
691
714
692
715
buf = kzalloc(sizeof(*buf),
693
716
gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
···
1056
1085
for_each_sg(sg, s, nents, i)
1057
1086
ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1058
1087
dir);
1059
-
}
1060
-
1061
-
/*
1062
-
* Return whether the given device DMA address mask can be supported
1063
-
* properly. For example, if your device can only drive the low 24-bits
1064
-
* during bus mastering, then you would pass 0x00ffffff as the mask
1065
-
* to this function.
1066
-
*/
1067
-
int arm_dma_supported(struct device *dev, u64 mask)
1068
-
{
1069
-
return __dma_supported(dev, mask, false);
1070
1088
}
1071
1089
1072
1090
static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
+1
-1
arch/microblaze/Kconfig
+1
-1
arch/microblaze/Kconfig
···
8
8
select ARCH_HAS_GCOV_PROFILE_ALL
9
9
select ARCH_HAS_SYNC_DMA_FOR_CPU
10
10
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
11
-
select ARCH_HAS_UNCACHED_SEGMENT if !MMU
11
+
select ARCH_HAS_DMA_SET_UNCACHED if !MMU
12
12
select ARCH_MIGHT_HAVE_PC_PARPORT
13
13
select ARCH_WANT_IPC_PARSE_VERSION
14
14
select BUILDTIME_TABLE_SORT
+1
-8
arch/microblaze/mm/consistent.c
+1
-8
arch/microblaze/mm/consistent.c
···
40
40
#define UNCACHED_SHADOW_MASK 0
41
41
#endif /* CONFIG_XILINX_UNCACHED_SHADOW */
42
42
43
-
void *uncached_kernel_address(void *ptr)
43
+
void *arch_dma_set_uncached(void *ptr, size_t size)
44
44
{
45
45
unsigned long addr = (unsigned long)ptr;
46
46
···
48
48
if (addr > cpuinfo.dcache_base && addr < cpuinfo.dcache_high)
49
49
pr_warn("ERROR: Your cache coherent area is CACHED!!!\n");
50
50
return (void *)addr;
51
-
}
52
-
53
-
void *cached_kernel_address(void *ptr)
54
-
{
55
-
unsigned long addr = (unsigned long)ptr;
56
-
57
-
return (void *)(addr & ~UNCACHED_SHADOW_MASK);
58
51
}
59
52
#endif /* CONFIG_MMU */
+2
-1
arch/mips/Kconfig
+2
-1
arch/mips/Kconfig
···
1192
1192
# significant advantages.
1193
1193
#
1194
1194
select ARCH_HAS_DMA_WRITE_COMBINE
1195
+
select ARCH_HAS_DMA_PREP_COHERENT
1195
1196
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
1196
-
select ARCH_HAS_UNCACHED_SEGMENT
1197
+
select ARCH_HAS_DMA_SET_UNCACHED
1197
1198
select DMA_NONCOHERENT_MMAP
1198
1199
select DMA_NONCOHERENT_CACHE_SYNC
1199
1200
select NEED_DMA_MAP_STATE
+1
-6
arch/mips/mm/dma-noncoherent.c
+1
-6
arch/mips/mm/dma-noncoherent.c
···
49
49
dma_cache_wback_inv((unsigned long)page_address(page), size);
50
50
}
51
51
52
-
void *uncached_kernel_address(void *addr)
52
+
void *arch_dma_set_uncached(void *addr, size_t size)
53
53
{
54
54
return (void *)(__pa(addr) + UNCAC_BASE);
55
-
}
56
-
57
-
void *cached_kernel_address(void *addr)
58
-
{
59
-
return __va(addr) - UNCAC_BASE;
60
55
}
61
56
62
57
static inline void dma_sync_virt(void *addr, size_t size,
+2
-1
arch/nios2/Kconfig
+2
-1
arch/nios2/Kconfig
···
2
2
config NIOS2
3
3
def_bool y
4
4
select ARCH_32BIT_OFF_T
5
+
select ARCH_HAS_DMA_PREP_COHERENT
5
6
select ARCH_HAS_SYNC_DMA_FOR_CPU
6
7
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
7
-
select ARCH_HAS_UNCACHED_SEGMENT
8
+
select ARCH_HAS_DMA_SET_UNCACHED
8
9
select ARCH_NO_SWAP
9
10
select TIMER_OF
10
11
select GENERIC_ATOMIC64
+1
-11
arch/nios2/mm/dma-mapping.c
+1
-11
arch/nios2/mm/dma-mapping.c
···
67
67
flush_dcache_range(start, start + size);
68
68
}
69
69
70
-
void *uncached_kernel_address(void *ptr)
70
+
void *arch_dma_set_uncached(void *ptr, size_t size)
71
71
{
72
72
unsigned long addr = (unsigned long)ptr;
73
73
74
74
addr |= CONFIG_NIOS2_IO_REGION_BASE;
75
-
76
-
return (void *)ptr;
77
-
}
78
-
79
-
void *cached_kernel_address(void *ptr)
80
-
{
81
-
unsigned long addr = (unsigned long)ptr;
82
-
83
-
addr &= ~CONFIG_NIOS2_IO_REGION_BASE;
84
-
addr |= CONFIG_NIOS2_KERNEL_REGION_BASE;
85
75
86
76
return (void *)ptr;
87
77
}
+2
arch/openrisc/Kconfig
+2
arch/openrisc/Kconfig
+10
-45
arch/openrisc/kernel/dma.c
+10
-45
arch/openrisc/kernel/dma.c
···
11
11
* Copyright (C) 2010-2011 Jonas Bonn <jonas@southpole.se>
12
12
*
13
13
* DMA mapping callbacks...
14
-
* As alloc_coherent is the only DMA callback being used currently, that's
15
-
* the only thing implemented properly. The rest need looking into...
16
14
*/
17
15
18
16
#include <linux/dma-noncoherent.h>
···
65
67
.pte_entry = page_clear_nocache,
66
68
};
67
69
68
-
/*
69
-
* Alloc "coherent" memory, which for OpenRISC means simply uncached.
70
-
*
71
-
* This function effectively just calls __get_free_pages, sets the
72
-
* cache-inhibit bit on those pages, and makes sure that the pages are
73
-
* flushed out of the cache before they are used.
74
-
*
75
-
* If the NON_CONSISTENT attribute is set, then this function just
76
-
* returns "normal", cachable memory.
77
-
*
78
-
* There are additional flags WEAK_ORDERING and WRITE_COMBINE to take
79
-
* into consideration here, too. All current known implementations of
80
-
* the OR1K support only strongly ordered memory accesses, so that flag
81
-
* is being ignored for now; uncached but write-combined memory is a
82
-
* missing feature of the OR1K.
83
-
*/
84
-
void *
85
-
arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
86
-
gfp_t gfp, unsigned long attrs)
70
+
void *arch_dma_set_uncached(void *cpu_addr, size_t size)
87
71
{
88
-
unsigned long va;
89
-
void *page;
90
-
91
-
page = alloc_pages_exact(size, gfp | __GFP_ZERO);
92
-
if (!page)
93
-
return NULL;
94
-
95
-
/* This gives us the real physical address of the first page. */
96
-
*dma_handle = __pa(page);
97
-
98
-
va = (unsigned long)page;
72
+
unsigned long va = (unsigned long)cpu_addr;
73
+
int error;
99
74
100
75
/*
101
76
* We need to iterate through the pages, clearing the dcache for
102
77
* them and setting the cache-inhibit bit.
103
78
*/
104
-
if (walk_page_range(&init_mm, va, va + size, &set_nocache_walk_ops,
105
-
NULL)) {
106
-
free_pages_exact(page, size);
107
-
return NULL;
108
-
}
109
-
110
-
return (void *)va;
79
+
error = walk_page_range(&init_mm, va, va + size, &set_nocache_walk_ops,
80
+
NULL);
81
+
if (error)
82
+
return ERR_PTR(error);
83
+
return cpu_addr;
111
84
}
112
85
113
-
void
114
-
arch_dma_free(struct device *dev, size_t size, void *vaddr,
115
-
dma_addr_t dma_handle, unsigned long attrs)
86
+
void arch_dma_clear_uncached(void *cpu_addr, size_t size)
116
87
{
117
-
unsigned long va = (unsigned long)vaddr;
88
+
unsigned long va = (unsigned long)cpu_addr;
118
89
119
90
/* walk_page_range shouldn't be able to fail here */
120
91
WARN_ON(walk_page_range(&init_mm, va, va + size,
121
92
&clear_nocache_walk_ops, NULL));
122
-
123
-
free_pages_exact(vaddr, size);
124
93
}
125
94
126
95
void arch_sync_dma_for_device(phys_addr_t addr, size_t size,
+1
-1
arch/xtensa/Kconfig
+1
-1
arch/xtensa/Kconfig
···
6
6
select ARCH_HAS_DMA_PREP_COHERENT if MMU
7
7
select ARCH_HAS_SYNC_DMA_FOR_CPU if MMU
8
8
select ARCH_HAS_SYNC_DMA_FOR_DEVICE if MMU
9
-
select ARCH_HAS_UNCACHED_SEGMENT if MMU
9
+
select ARCH_HAS_DMA_SET_UNCACHED if MMU
10
10
select ARCH_USE_QUEUED_RWLOCKS
11
11
select ARCH_USE_QUEUED_SPINLOCKS
12
12
select ARCH_WANT_FRAME_POINTERS
+3
-9
arch/xtensa/kernel/pci-dma.c
+3
-9
arch/xtensa/kernel/pci-dma.c
···
88
88
89
89
/*
90
90
* Memory caching is platform-dependent in noMMU xtensa configurations.
91
-
* The following two functions should be implemented in platform code
92
-
* in order to enable coherent DMA memory operations when CONFIG_MMU is not
93
-
* enabled.
91
+
* This function should be implemented in platform code in order to enable
92
+
* coherent DMA memory operations when CONFIG_MMU is not enabled.
94
93
*/
95
94
#ifdef CONFIG_MMU
96
-
void *uncached_kernel_address(void *p)
95
+
void *arch_dma_set_uncached(void *p, size_t size)
97
96
{
98
97
return p + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR;
99
-
}
100
-
101
-
void *cached_kernel_address(void *p)
102
-
{
103
-
return p + XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
104
98
}
105
99
#endif /* CONFIG_MMU */
+2
-2
include/linux/dma-noncoherent.h
+2
-2
include/linux/dma-noncoherent.h
···
108
108
}
109
109
#endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
110
110
111
-
void *uncached_kernel_address(void *addr);
112
-
void *cached_kernel_address(void *addr);
111
+
void *arch_dma_set_uncached(void *addr, size_t size);
112
+
void arch_dma_clear_uncached(void *addr, size_t size);
113
113
114
114
#endif /* _LINUX_DMA_NONCOHERENT_H */
+7
-6
kernel/dma/coherent.c
+7
-6
kernel/dma/coherent.c
···
134
134
135
135
spin_lock_irqsave(&mem->spinlock, flags);
136
136
137
-
if (unlikely(size > (mem->size << PAGE_SHIFT)))
137
+
if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))
138
138
goto err;
139
139
140
140
pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
···
144
144
/*
145
145
* Memory was found in the coherent area.
146
146
*/
147
-
*dma_handle = dma_get_device_base(dev, mem) + (pageno << PAGE_SHIFT);
148
-
ret = mem->virt_base + (pageno << PAGE_SHIFT);
147
+
*dma_handle = dma_get_device_base(dev, mem) +
148
+
((dma_addr_t)pageno << PAGE_SHIFT);
149
+
ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);
149
150
spin_unlock_irqrestore(&mem->spinlock, flags);
150
151
memset(ret, 0, size);
151
152
return ret;
···
195
194
int order, void *vaddr)
196
195
{
197
196
if (mem && vaddr >= mem->virt_base && vaddr <
198
-
(mem->virt_base + (mem->size << PAGE_SHIFT))) {
197
+
(mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {
199
198
int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
200
199
unsigned long flags;
201
200
···
239
238
struct vm_area_struct *vma, void *vaddr, size_t size, int *ret)
240
239
{
241
240
if (mem && vaddr >= mem->virt_base && vaddr + size <=
242
-
(mem->virt_base + (mem->size << PAGE_SHIFT))) {
241
+
(mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {
243
242
unsigned long off = vma->vm_pgoff;
244
243
int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
245
-
int user_count = vma_pages(vma);
244
+
unsigned long user_count = vma_pages(vma);
246
245
int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
247
246
248
247
*ret = -ENXIO;
+14
-11
kernel/dma/direct.c
+14
-11
kernel/dma/direct.c
···
157
157
ret = dma_common_contiguous_remap(page, PAGE_ALIGN(size),
158
158
dma_pgprot(dev, PAGE_KERNEL, attrs),
159
159
__builtin_return_address(0));
160
-
if (!ret) {
161
-
dma_free_contiguous(dev, page, size);
162
-
return ret;
163
-
}
164
-
160
+
if (!ret)
161
+
goto out_free_pages;
165
162
memset(ret, 0, size);
166
163
goto done;
167
164
}
···
171
174
* so log an error and fail.
172
175
*/
173
176
dev_info(dev, "Rejecting highmem page from CMA.\n");
174
-
dma_free_contiguous(dev, page, size);
175
-
return NULL;
177
+
goto out_free_pages;
176
178
}
177
179
178
180
ret = page_address(page);
···
180
184
181
185
memset(ret, 0, size);
182
186
183
-
if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
187
+
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
184
188
dma_alloc_need_uncached(dev, attrs)) {
185
189
arch_dma_prep_coherent(page, size);
186
-
ret = uncached_kernel_address(ret);
190
+
ret = arch_dma_set_uncached(ret, size);
191
+
if (IS_ERR(ret))
192
+
goto out_free_pages;
187
193
}
188
194
done:
189
195
if (force_dma_unencrypted(dev))
···
193
195
else
194
196
*dma_handle = phys_to_dma(dev, page_to_phys(page));
195
197
return ret;
198
+
out_free_pages:
199
+
dma_free_contiguous(dev, page, size);
200
+
return NULL;
196
201
}
197
202
198
203
void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
···
219
218
220
219
if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr))
221
220
vunmap(cpu_addr);
221
+
else if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
222
+
arch_dma_clear_uncached(cpu_addr, size);
222
223
223
224
dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
224
225
}
···
228
225
void *dma_direct_alloc(struct device *dev, size_t size,
229
226
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
230
227
{
231
-
if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
228
+
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
232
229
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
233
230
dma_alloc_need_uncached(dev, attrs))
234
231
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
···
238
235
void dma_direct_free(struct device *dev, size_t size,
239
236
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
240
237
{
241
-
if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
238
+
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
242
239
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
243
240
dma_alloc_need_uncached(dev, attrs))
244
241
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);