Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping updates from Christoph Hellwig:
- make support for dma_ops optional
- move more code out of line
- add generic support for a dma_ops bypass mode
- misc cleanups
* tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping:
dma-contiguous: cleanup dma_alloc_contiguous
dma-debug: use named initializers for dir2name
powerpc: use the generic dma_ops_bypass mode
dma-mapping: add a dma_ops_bypass flag to struct device
dma-mapping: make support for dma ops optional
dma-mapping: inline the fast path dma-direct calls
dma-mapping: move the remaining DMA API calls out of line
+415
-416
+1
arch/alpha/Kconfig
+1
arch/alpha/Kconfig
+1
arch/arm/Kconfig
+1
arch/arm/Kconfig
+1
arch/ia64/Kconfig
+1
arch/ia64/Kconfig
+1
arch/mips/Kconfig
+1
arch/mips/Kconfig
+1
arch/parisc/Kconfig
+1
arch/parisc/Kconfig
+2
arch/powerpc/Kconfig
+2
arch/powerpc/Kconfig
···
151
151
select BUILDTIME_TABLE_SORT
152
152
select CLONE_BACKWARDS
153
153
select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
154
+
select DMA_OPS if PPC64
155
+
select DMA_OPS_BYPASS if PPC64
154
156
select DYNAMIC_FTRACE if FUNCTION_TRACER
155
157
select EDAC_ATOMIC_SCRUB
156
158
select EDAC_SUPPORT
-5
arch/powerpc/include/asm/device.h
-5
arch/powerpc/include/asm/device.h
···
19
19
*/
20
20
struct dev_archdata {
21
21
/*
22
-
* Set to %true if the dma_iommu_ops are requested to use a direct
23
-
* window instead of dynamically mapping memory.
24
-
*/
25
-
bool iommu_bypass : 1;
26
-
/*
27
22
* These two used to be a union. However, with the hybrid ops we need
28
23
* both so here we store both a DMA offset for direct mappings and
29
24
* an iommu_table for remapped DMA.
+9
-81
arch/powerpc/kernel/dma-iommu.c
+9
-81
arch/powerpc/kernel/dma-iommu.c
···
14
14
* Generic iommu implementation
15
15
*/
16
16
17
-
/*
18
-
* The coherent mask may be smaller than the real mask, check if we can
19
-
* really use a direct window.
20
-
*/
21
-
static inline bool dma_iommu_alloc_bypass(struct device *dev)
22
-
{
23
-
return dev->archdata.iommu_bypass && !iommu_fixed_is_weak &&
24
-
dma_direct_supported(dev, dev->coherent_dma_mask);
25
-
}
26
-
27
-
static inline bool dma_iommu_map_bypass(struct device *dev,
28
-
unsigned long attrs)
29
-
{
30
-
return dev->archdata.iommu_bypass &&
31
-
(!iommu_fixed_is_weak || (attrs & DMA_ATTR_WEAK_ORDERING));
32
-
}
33
-
34
17
/* Allocates a contiguous real buffer and creates mappings over it.
35
18
* Returns the virtual address of the buffer and sets dma_handle
36
19
* to the dma address (mapping) of the first page.
···
22
39
dma_addr_t *dma_handle, gfp_t flag,
23
40
unsigned long attrs)
24
41
{
25
-
if (dma_iommu_alloc_bypass(dev))
26
-
return dma_direct_alloc(dev, size, dma_handle, flag, attrs);
27
42
return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size,
28
43
dma_handle, dev->coherent_dma_mask, flag,
29
44
dev_to_node(dev));
···
31
50
void *vaddr, dma_addr_t dma_handle,
32
51
unsigned long attrs)
33
52
{
34
-
if (dma_iommu_alloc_bypass(dev))
35
-
dma_direct_free(dev, size, vaddr, dma_handle, attrs);
36
-
else
37
-
iommu_free_coherent(get_iommu_table_base(dev), size, vaddr,
38
-
dma_handle);
53
+
iommu_free_coherent(get_iommu_table_base(dev), size, vaddr, dma_handle);
39
54
}
40
55
41
56
/* Creates TCEs for a user provided buffer. The user buffer must be
···
44
67
enum dma_data_direction direction,
45
68
unsigned long attrs)
46
69
{
47
-
if (dma_iommu_map_bypass(dev, attrs))
48
-
return dma_direct_map_page(dev, page, offset, size, direction,
49
-
attrs);
50
70
return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
51
71
size, dma_get_mask(dev), direction, attrs);
52
72
}
···
53
79
size_t size, enum dma_data_direction direction,
54
80
unsigned long attrs)
55
81
{
56
-
if (!dma_iommu_map_bypass(dev, attrs))
57
-
iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size,
58
-
direction, attrs);
59
-
else
60
-
dma_direct_unmap_page(dev, dma_handle, size, direction, attrs);
82
+
iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction,
83
+
attrs);
61
84
}
62
85
63
86
···
62
91
int nelems, enum dma_data_direction direction,
63
92
unsigned long attrs)
64
93
{
65
-
if (dma_iommu_map_bypass(dev, attrs))
66
-
return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
67
94
return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
68
95
dma_get_mask(dev), direction, attrs);
69
96
}
···
70
101
int nelems, enum dma_data_direction direction,
71
102
unsigned long attrs)
72
103
{
73
-
if (!dma_iommu_map_bypass(dev, attrs))
74
-
ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
104
+
ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
75
105
direction, attrs);
76
-
else
77
-
dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
78
106
}
79
107
80
108
static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)
···
79
113
struct pci_dev *pdev = to_pci_dev(dev);
80
114
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
81
115
82
-
return phb->controller_ops.iommu_bypass_supported &&
83
-
phb->controller_ops.iommu_bypass_supported(pdev, mask);
116
+
if (iommu_fixed_is_weak || !phb->controller_ops.iommu_bypass_supported)
117
+
return false;
118
+
return phb->controller_ops.iommu_bypass_supported(pdev, mask);
84
119
}
85
120
86
121
/* We support DMA to/from any memory page via the iommu */
···
90
123
struct iommu_table *tbl = get_iommu_table_base(dev);
91
124
92
125
if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
93
-
dev->archdata.iommu_bypass = true;
126
+
dev->dma_ops_bypass = true;
94
127
dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
95
128
return 1;
96
129
}
···
108
141
}
109
142
110
143
dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
111
-
dev->archdata.iommu_bypass = false;
144
+
dev->dma_ops_bypass = false;
112
145
return 1;
113
146
}
114
147
···
120
153
if (!tbl)
121
154
return 0;
122
155
123
-
if (dev_is_pci(dev)) {
124
-
u64 bypass_mask = dma_direct_get_required_mask(dev);
125
-
126
-
if (dma_iommu_bypass_supported(dev, bypass_mask))
127
-
return bypass_mask;
128
-
}
129
-
130
156
mask = 1ULL < (fls_long(tbl->it_offset + tbl->it_size) - 1);
131
157
mask += mask - 1;
132
158
133
159
return mask;
134
-
}
135
-
136
-
static void dma_iommu_sync_for_cpu(struct device *dev, dma_addr_t addr,
137
-
size_t size, enum dma_data_direction dir)
138
-
{
139
-
if (dma_iommu_alloc_bypass(dev))
140
-
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
141
-
}
142
-
143
-
static void dma_iommu_sync_for_device(struct device *dev, dma_addr_t addr,
144
-
size_t sz, enum dma_data_direction dir)
145
-
{
146
-
if (dma_iommu_alloc_bypass(dev))
147
-
dma_direct_sync_single_for_device(dev, addr, sz, dir);
148
-
}
149
-
150
-
extern void dma_iommu_sync_sg_for_cpu(struct device *dev,
151
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
152
-
{
153
-
if (dma_iommu_alloc_bypass(dev))
154
-
dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
155
-
}
156
-
157
-
extern void dma_iommu_sync_sg_for_device(struct device *dev,
158
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
159
-
{
160
-
if (dma_iommu_alloc_bypass(dev))
161
-
dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
162
160
}
163
161
164
162
const struct dma_map_ops dma_iommu_ops = {
···
135
203
.map_page = dma_iommu_map_page,
136
204
.unmap_page = dma_iommu_unmap_page,
137
205
.get_required_mask = dma_iommu_get_required_mask,
138
-
.sync_single_for_cpu = dma_iommu_sync_for_cpu,
139
-
.sync_single_for_device = dma_iommu_sync_for_device,
140
-
.sync_sg_for_cpu = dma_iommu_sync_sg_for_cpu,
141
-
.sync_sg_for_device = dma_iommu_sync_sg_for_device,
142
206
.mmap = dma_common_mmap,
143
207
.get_sgtable = dma_common_get_sgtable,
144
208
};
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
+1
arch/sparc/Kconfig
+1
arch/sparc/Kconfig
+1
arch/x86/Kconfig
+1
arch/x86/Kconfig
+5
-1
drivers/infiniband/core/device.c
+5
-1
drivers/infiniband/core/device.c
···
1183
1183
struct device *parent = device->dev.parent;
1184
1184
1185
1185
WARN_ON_ONCE(device->dma_device);
1186
+
1187
+
#ifdef CONFIG_DMA_OPS
1186
1188
if (device->dev.dma_ops) {
1187
1189
/*
1188
1190
* The caller provided custom DMA operations. Copy the
···
1205
1203
else
1206
1204
WARN_ON_ONCE(true);
1207
1205
}
1208
-
} else {
1206
+
} else
1207
+
#endif /* CONFIG_DMA_OPS */
1208
+
{
1209
1209
/*
1210
1210
* The caller did not provide custom DMA operations. Use the
1211
1211
* DMA mapping operations of the parent device.
+2
drivers/iommu/Kconfig
+2
drivers/iommu/Kconfig
···
97
97
# IOMMU-agnostic DMA-mapping layer
98
98
config IOMMU_DMA
99
99
bool
100
+
select DMA_OPS
100
101
select IOMMU_API
101
102
select IOMMU_IOVA
102
103
select IRQ_MSI_IOMMU
···
184
183
config INTEL_IOMMU
185
184
bool "Support for Intel IOMMU using DMA Remapping Devices"
186
185
depends on PCI_MSI && ACPI && (X86 || IA64)
186
+
select DMA_OPS
187
187
select IOMMU_API
188
188
select IOMMU_IOVA
189
189
select NEED_DMA_MAP_STATE
+2
-2
drivers/macintosh/macio_asic.c
+2
-2
drivers/macintosh/macio_asic.c
···
382
382
dma_set_max_seg_size(&dev->ofdev.dev, 65536);
383
383
dma_set_seg_boundary(&dev->ofdev.dev, 0xffffffff);
384
384
385
-
#ifdef CONFIG_PCI
385
+
#if defined(CONFIG_PCI) && defined(CONFIG_DMA_OPS)
386
386
/* Set the DMA ops to the ones from the PCI device, this could be
387
387
* fishy if we didn't know that on PowerMac it's always direct ops
388
388
* or iommu ops that will work fine
···
391
391
*/
392
392
dev->ofdev.dev.archdata = chip->lbus.pdev->dev.archdata;
393
393
dev->ofdev.dev.dma_ops = chip->lbus.pdev->dev.dma_ops;
394
-
#endif /* CONFIG_PCI */
394
+
#endif /* CONFIG_PCI && CONFIG_DMA_OPS */
395
395
396
396
#ifdef DEBUG
397
397
printk("preparing mdev @%p, ofdev @%p, dev @%p, kobj @%p\n",
+4
drivers/misc/mic/Kconfig
+4
drivers/misc/mic/Kconfig
···
4
4
config INTEL_MIC_BUS
5
5
tristate "Intel MIC Bus Driver"
6
6
depends on 64BIT && PCI && X86
7
+
select DMA_OPS
7
8
help
8
9
This option is selected by any driver which registers a
9
10
device or driver on the MIC Bus, such as CONFIG_INTEL_MIC_HOST,
···
20
19
config SCIF_BUS
21
20
tristate "SCIF Bus Driver"
22
21
depends on 64BIT && PCI && X86
22
+
select DMA_OPS
23
23
help
24
24
This option is selected by any driver which registers a
25
25
device or driver on the SCIF Bus, such as CONFIG_INTEL_MIC_HOST
···
35
33
36
34
config VOP_BUS
37
35
tristate "VOP Bus Driver"
36
+
select DMA_OPS
38
37
help
39
38
This option is selected by any driver which registers a
40
39
device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
···
52
49
tristate "Intel MIC Host Driver"
53
50
depends on 64BIT && PCI && X86
54
51
depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS
52
+
select DMA_OPS
55
53
help
56
54
This enables Host Driver support for the Intel Many Integrated
57
55
Core (MIC) family of PCIe form factor coprocessor devices that
+1
drivers/vdpa/Kconfig
+1
drivers/vdpa/Kconfig
+1
drivers/xen/Kconfig
+1
drivers/xen/Kconfig
+10
-1
include/linux/device.h
+10
-1
include/linux/device.h
···
525
525
* sync_state() callback.
526
526
* @dma_coherent: this particular device is dma coherent, even if the
527
527
* architecture supports non-coherent devices.
528
+
* @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
529
+
* streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
530
+
* and optionall (if the coherent mask is large enough) also
531
+
* for dma allocations. This flag is managed by the dma ops
532
+
* instance from ->dma_supported.
528
533
*
529
534
* At the lowest level, every device in a Linux system is represented by an
530
535
* instance of struct device. The device structure contains the information
···
579
574
#ifdef CONFIG_GENERIC_MSI_IRQ
580
575
struct list_head msi_list;
581
576
#endif
582
-
577
+
#ifdef CONFIG_DMA_OPS
583
578
const struct dma_map_ops *dma_ops;
579
+
#endif
584
580
u64 *dma_mask; /* dma mask (if dma'able device) */
585
581
u64 coherent_dma_mask;/* Like dma_mask, but for
586
582
alloc_coherent mappings as
···
633
627
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
634
628
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
635
629
bool dma_coherent:1;
630
+
#endif
631
+
#ifdef CONFIG_DMA_OPS_BYPASS
632
+
bool dma_ops_bypass : 1;
636
633
#endif
637
634
};
638
635
+104
include/linux/dma-direct.h
+104
include/linux/dma-direct.h
···
1
1
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Internals of the DMA direct mapping implementation. Only for use by the
4
+
* DMA mapping code and IOMMU drivers.
5
+
*/
2
6
#ifndef _LINUX_DMA_DIRECT_H
3
7
#define _LINUX_DMA_DIRECT_H 1
4
8
5
9
#include <linux/dma-mapping.h>
10
+
#include <linux/dma-noncoherent.h>
6
11
#include <linux/memblock.h> /* for min_low_pfn */
7
12
#include <linux/mem_encrypt.h>
13
+
#include <linux/swiotlb.h>
8
14
9
15
extern unsigned int zone_dma_bits;
10
16
···
93
87
unsigned long attrs);
94
88
int dma_direct_supported(struct device *dev, u64 mask);
95
89
bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
90
+
int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
91
+
enum dma_data_direction dir, unsigned long attrs);
92
+
dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
93
+
size_t size, enum dma_data_direction dir, unsigned long attrs);
94
+
size_t dma_direct_max_mapping_size(struct device *dev);
95
+
96
+
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
97
+
defined(CONFIG_SWIOTLB)
98
+
void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
99
+
int nents, enum dma_data_direction dir);
100
+
#else
101
+
static inline void dma_direct_sync_sg_for_device(struct device *dev,
102
+
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
103
+
{
104
+
}
105
+
#endif
106
+
107
+
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
108
+
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
109
+
defined(CONFIG_SWIOTLB)
110
+
void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
111
+
int nents, enum dma_data_direction dir, unsigned long attrs);
112
+
void dma_direct_sync_sg_for_cpu(struct device *dev,
113
+
struct scatterlist *sgl, int nents, enum dma_data_direction dir);
114
+
#else
115
+
static inline void dma_direct_unmap_sg(struct device *dev,
116
+
struct scatterlist *sgl, int nents, enum dma_data_direction dir,
117
+
unsigned long attrs)
118
+
{
119
+
}
120
+
static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
121
+
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
122
+
{
123
+
}
124
+
#endif
125
+
126
+
static inline void dma_direct_sync_single_for_device(struct device *dev,
127
+
dma_addr_t addr, size_t size, enum dma_data_direction dir)
128
+
{
129
+
phys_addr_t paddr = dma_to_phys(dev, addr);
130
+
131
+
if (unlikely(is_swiotlb_buffer(paddr)))
132
+
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
133
+
134
+
if (!dev_is_dma_coherent(dev))
135
+
arch_sync_dma_for_device(paddr, size, dir);
136
+
}
137
+
138
+
static inline void dma_direct_sync_single_for_cpu(struct device *dev,
139
+
dma_addr_t addr, size_t size, enum dma_data_direction dir)
140
+
{
141
+
phys_addr_t paddr = dma_to_phys(dev, addr);
142
+
143
+
if (!dev_is_dma_coherent(dev)) {
144
+
arch_sync_dma_for_cpu(paddr, size, dir);
145
+
arch_sync_dma_for_cpu_all();
146
+
}
147
+
148
+
if (unlikely(is_swiotlb_buffer(paddr)))
149
+
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
150
+
}
151
+
152
+
static inline dma_addr_t dma_direct_map_page(struct device *dev,
153
+
struct page *page, unsigned long offset, size_t size,
154
+
enum dma_data_direction dir, unsigned long attrs)
155
+
{
156
+
phys_addr_t phys = page_to_phys(page) + offset;
157
+
dma_addr_t dma_addr = phys_to_dma(dev, phys);
158
+
159
+
if (unlikely(swiotlb_force == SWIOTLB_FORCE))
160
+
return swiotlb_map(dev, phys, size, dir, attrs);
161
+
162
+
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
163
+
if (swiotlb_force != SWIOTLB_NO_FORCE)
164
+
return swiotlb_map(dev, phys, size, dir, attrs);
165
+
166
+
dev_WARN_ONCE(dev, 1,
167
+
"DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
168
+
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
169
+
return DMA_MAPPING_ERROR;
170
+
}
171
+
172
+
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
173
+
arch_sync_dma_for_device(phys, size, dir);
174
+
return dma_addr;
175
+
}
176
+
177
+
static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
178
+
size_t size, enum dma_data_direction dir, unsigned long attrs)
179
+
{
180
+
phys_addr_t phys = dma_to_phys(dev, addr);
181
+
182
+
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
183
+
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
184
+
185
+
if (unlikely(is_swiotlb_buffer(phys)))
186
+
swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
187
+
}
96
188
#endif /* _LINUX_DMA_DIRECT_H */
+29
-222
include/linux/dma-mapping.h
+29
-222
include/linux/dma-mapping.h
···
188
188
}
189
189
#endif /* CONFIG_DMA_DECLARE_COHERENT */
190
190
191
-
static inline bool dma_is_direct(const struct dma_map_ops *ops)
192
-
{
193
-
return likely(!ops);
194
-
}
195
-
196
-
/*
197
-
* All the dma_direct_* declarations are here just for the indirect call bypass,
198
-
* and must not be used directly drivers!
199
-
*/
200
-
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
201
-
unsigned long offset, size_t size, enum dma_data_direction dir,
202
-
unsigned long attrs);
203
-
int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
204
-
enum dma_data_direction dir, unsigned long attrs);
205
-
dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
206
-
size_t size, enum dma_data_direction dir, unsigned long attrs);
207
-
208
-
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
209
-
defined(CONFIG_SWIOTLB)
210
-
void dma_direct_sync_single_for_device(struct device *dev,
211
-
dma_addr_t addr, size_t size, enum dma_data_direction dir);
212
-
void dma_direct_sync_sg_for_device(struct device *dev,
213
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir);
214
-
#else
215
-
static inline void dma_direct_sync_single_for_device(struct device *dev,
216
-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
217
-
{
218
-
}
219
-
static inline void dma_direct_sync_sg_for_device(struct device *dev,
220
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
221
-
{
222
-
}
223
-
#endif
224
-
225
-
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
226
-
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
227
-
defined(CONFIG_SWIOTLB)
228
-
void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
229
-
size_t size, enum dma_data_direction dir, unsigned long attrs);
230
-
void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
231
-
int nents, enum dma_data_direction dir, unsigned long attrs);
232
-
void dma_direct_sync_single_for_cpu(struct device *dev,
233
-
dma_addr_t addr, size_t size, enum dma_data_direction dir);
234
-
void dma_direct_sync_sg_for_cpu(struct device *dev,
235
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir);
236
-
#else
237
-
static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238
-
size_t size, enum dma_data_direction dir, unsigned long attrs)
239
-
{
240
-
}
241
-
static inline void dma_direct_unmap_sg(struct device *dev,
242
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir,
243
-
unsigned long attrs)
244
-
{
245
-
}
246
-
static inline void dma_direct_sync_single_for_cpu(struct device *dev,
247
-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
248
-
{
249
-
}
250
-
static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
251
-
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
252
-
{
253
-
}
254
-
#endif
255
-
256
-
size_t dma_direct_max_mapping_size(struct device *dev);
257
-
258
191
#ifdef CONFIG_HAS_DMA
259
192
#include <asm/dma-mapping.h>
260
193
194
+
#ifdef CONFIG_DMA_OPS
261
195
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
262
196
{
263
197
if (dev->dma_ops)
···
204
270
{
205
271
dev->dma_ops = dma_ops;
206
272
}
207
-
208
-
static inline dma_addr_t dma_map_page_attrs(struct device *dev,
209
-
struct page *page, size_t offset, size_t size,
210
-
enum dma_data_direction dir, unsigned long attrs)
273
+
#else /* CONFIG_DMA_OPS */
274
+
static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
211
275
{
212
-
const struct dma_map_ops *ops = get_dma_ops(dev);
213
-
dma_addr_t addr;
214
-
215
-
BUG_ON(!valid_dma_direction(dir));
216
-
if (dma_is_direct(ops))
217
-
addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
218
-
else
219
-
addr = ops->map_page(dev, page, offset, size, dir, attrs);
220
-
debug_dma_map_page(dev, page, offset, size, dir, addr);
221
-
222
-
return addr;
276
+
return NULL;
223
277
}
224
-
225
-
static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
226
-
size_t size, enum dma_data_direction dir, unsigned long attrs)
278
+
static inline void set_dma_ops(struct device *dev,
279
+
const struct dma_map_ops *dma_ops)
227
280
{
228
-
const struct dma_map_ops *ops = get_dma_ops(dev);
229
-
230
-
BUG_ON(!valid_dma_direction(dir));
231
-
if (dma_is_direct(ops))
232
-
dma_direct_unmap_page(dev, addr, size, dir, attrs);
233
-
else if (ops->unmap_page)
234
-
ops->unmap_page(dev, addr, size, dir, attrs);
235
-
debug_dma_unmap_page(dev, addr, size, dir);
236
281
}
237
-
238
-
/*
239
-
* dma_maps_sg_attrs returns 0 on error and > 0 on success.
240
-
* It should never return a value < 0.
241
-
*/
242
-
static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
243
-
int nents, enum dma_data_direction dir,
244
-
unsigned long attrs)
245
-
{
246
-
const struct dma_map_ops *ops = get_dma_ops(dev);
247
-
int ents;
248
-
249
-
BUG_ON(!valid_dma_direction(dir));
250
-
if (dma_is_direct(ops))
251
-
ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
252
-
else
253
-
ents = ops->map_sg(dev, sg, nents, dir, attrs);
254
-
BUG_ON(ents < 0);
255
-
debug_dma_map_sg(dev, sg, nents, ents, dir);
256
-
257
-
return ents;
258
-
}
259
-
260
-
static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
261
-
int nents, enum dma_data_direction dir,
262
-
unsigned long attrs)
263
-
{
264
-
const struct dma_map_ops *ops = get_dma_ops(dev);
265
-
266
-
BUG_ON(!valid_dma_direction(dir));
267
-
debug_dma_unmap_sg(dev, sg, nents, dir);
268
-
if (dma_is_direct(ops))
269
-
dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
270
-
else if (ops->unmap_sg)
271
-
ops->unmap_sg(dev, sg, nents, dir, attrs);
272
-
}
273
-
274
-
static inline dma_addr_t dma_map_resource(struct device *dev,
275
-
phys_addr_t phys_addr,
276
-
size_t size,
277
-
enum dma_data_direction dir,
278
-
unsigned long attrs)
279
-
{
280
-
const struct dma_map_ops *ops = get_dma_ops(dev);
281
-
dma_addr_t addr = DMA_MAPPING_ERROR;
282
-
283
-
BUG_ON(!valid_dma_direction(dir));
284
-
285
-
/* Don't allow RAM to be mapped */
286
-
if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
287
-
return DMA_MAPPING_ERROR;
288
-
289
-
if (dma_is_direct(ops))
290
-
addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
291
-
else if (ops->map_resource)
292
-
addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
293
-
294
-
debug_dma_map_resource(dev, phys_addr, size, dir, addr);
295
-
return addr;
296
-
}
297
-
298
-
static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
299
-
size_t size, enum dma_data_direction dir,
300
-
unsigned long attrs)
301
-
{
302
-
const struct dma_map_ops *ops = get_dma_ops(dev);
303
-
304
-
BUG_ON(!valid_dma_direction(dir));
305
-
if (!dma_is_direct(ops) && ops->unmap_resource)
306
-
ops->unmap_resource(dev, addr, size, dir, attrs);
307
-
debug_dma_unmap_resource(dev, addr, size, dir);
308
-
}
309
-
310
-
static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
311
-
size_t size,
312
-
enum dma_data_direction dir)
313
-
{
314
-
const struct dma_map_ops *ops = get_dma_ops(dev);
315
-
316
-
BUG_ON(!valid_dma_direction(dir));
317
-
if (dma_is_direct(ops))
318
-
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
319
-
else if (ops->sync_single_for_cpu)
320
-
ops->sync_single_for_cpu(dev, addr, size, dir);
321
-
debug_dma_sync_single_for_cpu(dev, addr, size, dir);
322
-
}
323
-
324
-
static inline void dma_sync_single_for_device(struct device *dev,
325
-
dma_addr_t addr, size_t size,
326
-
enum dma_data_direction dir)
327
-
{
328
-
const struct dma_map_ops *ops = get_dma_ops(dev);
329
-
330
-
BUG_ON(!valid_dma_direction(dir));
331
-
if (dma_is_direct(ops))
332
-
dma_direct_sync_single_for_device(dev, addr, size, dir);
333
-
else if (ops->sync_single_for_device)
334
-
ops->sync_single_for_device(dev, addr, size, dir);
335
-
debug_dma_sync_single_for_device(dev, addr, size, dir);
336
-
}
337
-
338
-
static inline void
339
-
dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
340
-
int nelems, enum dma_data_direction dir)
341
-
{
342
-
const struct dma_map_ops *ops = get_dma_ops(dev);
343
-
344
-
BUG_ON(!valid_dma_direction(dir));
345
-
if (dma_is_direct(ops))
346
-
dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
347
-
else if (ops->sync_sg_for_cpu)
348
-
ops->sync_sg_for_cpu(dev, sg, nelems, dir);
349
-
debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
350
-
}
351
-
352
-
static inline void
353
-
dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
354
-
int nelems, enum dma_data_direction dir)
355
-
{
356
-
const struct dma_map_ops *ops = get_dma_ops(dev);
357
-
358
-
BUG_ON(!valid_dma_direction(dir));
359
-
if (dma_is_direct(ops))
360
-
dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
361
-
else if (ops->sync_sg_for_device)
362
-
ops->sync_sg_for_device(dev, sg, nelems, dir);
363
-
debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
364
-
365
-
}
282
+
#endif /* CONFIG_DMA_OPS */
366
283
367
284
static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
368
285
{
···
224
439
return 0;
225
440
}
226
441
442
+
dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
443
+
size_t offset, size_t size, enum dma_data_direction dir,
444
+
unsigned long attrs);
445
+
void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
446
+
enum dma_data_direction dir, unsigned long attrs);
447
+
int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
448
+
enum dma_data_direction dir, unsigned long attrs);
449
+
void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
450
+
int nents, enum dma_data_direction dir,
451
+
unsigned long attrs);
452
+
dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
453
+
size_t size, enum dma_data_direction dir, unsigned long attrs);
454
+
void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
455
+
enum dma_data_direction dir, unsigned long attrs);
456
+
void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
457
+
enum dma_data_direction dir);
458
+
void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
459
+
size_t size, enum dma_data_direction dir);
460
+
void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
461
+
int nelems, enum dma_data_direction dir);
462
+
void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
463
+
int nelems, enum dma_data_direction dir);
227
464
void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
228
465
gfp_t flag, unsigned long attrs);
229
466
void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+12
kernel/dma/Kconfig
+12
kernel/dma/Kconfig
···
5
5
depends on !NO_DMA
6
6
default y
7
7
8
+
config DMA_OPS
9
+
bool
10
+
11
+
#
12
+
# IOMMU drivers that can bypass the IOMMU code and optionally use the direct
13
+
# mapping fast path should select this option and set the dma_ops_bypass
14
+
# flag in struct device where applicable
15
+
#
16
+
config DMA_OPS_BYPASS
17
+
bool
18
+
8
19
config NEED_SG_DMA_LENGTH
9
20
bool
10
21
···
71
60
config DMA_VIRT_OPS
72
61
bool
73
62
depends on HAS_DMA
63
+
select DMA_OPS
74
64
75
65
config SWIOTLB
76
66
bool
+2
-1
kernel/dma/Makefile
+2
-1
kernel/dma/Makefile
···
1
1
# SPDX-License-Identifier: GPL-2.0
2
2
3
-
obj-$(CONFIG_HAS_DMA) += mapping.o direct.o dummy.o
3
+
obj-$(CONFIG_HAS_DMA) += mapping.o direct.o
4
+
obj-$(CONFIG_DMA_OPS) += dummy.o
4
5
obj-$(CONFIG_DMA_CMA) += contiguous.o
5
6
obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o
6
7
obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
+14
-17
kernel/dma/contiguous.c
+14
-17
kernel/dma/contiguous.c
···
215
215
return cma_release(dev_get_cma_area(dev), pages, count);
216
216
}
217
217
218
+
static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
219
+
{
220
+
unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT);
221
+
222
+
return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN);
223
+
}
224
+
218
225
/**
219
226
* dma_alloc_contiguous() - allocate contiguous pages
220
227
* @dev: Pointer to device for which the allocation is performed.
···
238
231
*/
239
232
struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
240
233
{
241
-
size_t count = size >> PAGE_SHIFT;
242
-
struct page *page = NULL;
243
-
struct cma *cma = NULL;
244
-
245
-
if (dev && dev->cma_area)
246
-
cma = dev->cma_area;
247
-
else if (count > 1)
248
-
cma = dma_contiguous_default_area;
249
-
250
234
/* CMA can be used only in the context which permits sleeping */
251
-
if (cma && gfpflags_allow_blocking(gfp)) {
252
-
size_t align = get_order(size);
253
-
size_t cma_align = min_t(size_t, align, CONFIG_CMA_ALIGNMENT);
254
-
255
-
page = cma_alloc(cma, count, cma_align, gfp & __GFP_NOWARN);
256
-
}
257
-
258
-
return page;
235
+
if (!gfpflags_allow_blocking(gfp))
236
+
return NULL;
237
+
if (dev->cma_area)
238
+
return cma_alloc_aligned(dev->cma_area, size, gfp);
239
+
if (size <= PAGE_SIZE || !dma_contiguous_default_area)
240
+
return NULL;
241
+
return cma_alloc_aligned(dma_contiguous_default_area, size, gfp);
259
242
}
260
243
261
244
/**
+6
-2
kernel/dma/debug.c
+6
-2
kernel/dma/debug.c
···
144
144
[dma_debug_resource] = "resource",
145
145
};
146
146
147
-
static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
148
-
"DMA_FROM_DEVICE", "DMA_NONE" };
147
+
static const char *dir2name[] = {
148
+
[DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL",
149
+
[DMA_TO_DEVICE] = "DMA_TO_DEVICE",
150
+
[DMA_FROM_DEVICE] = "DMA_FROM_DEVICE",
151
+
[DMA_NONE] = "DMA_NONE",
152
+
};
149
153
150
154
/*
151
155
* The access to some variables in this macro is racy. We can't use atomic_t
-74
kernel/dma/direct.c
-74
kernel/dma/direct.c
···
10
10
#include <linux/dma-direct.h>
11
11
#include <linux/scatterlist.h>
12
12
#include <linux/dma-contiguous.h>
13
-
#include <linux/dma-noncoherent.h>
14
13
#include <linux/pfn.h>
15
14
#include <linux/vmalloc.h>
16
15
#include <linux/set_memory.h>
17
-
#include <linux/swiotlb.h>
18
16
19
17
/*
20
18
* Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it
···
302
304
303
305
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
304
306
defined(CONFIG_SWIOTLB)
305
-
void dma_direct_sync_single_for_device(struct device *dev,
306
-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
307
-
{
308
-
phys_addr_t paddr = dma_to_phys(dev, addr);
309
-
310
-
if (unlikely(is_swiotlb_buffer(paddr)))
311
-
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
312
-
313
-
if (!dev_is_dma_coherent(dev))
314
-
arch_sync_dma_for_device(paddr, size, dir);
315
-
}
316
-
EXPORT_SYMBOL(dma_direct_sync_single_for_device);
317
-
318
307
void dma_direct_sync_sg_for_device(struct device *dev,
319
308
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
320
309
{
···
320
335
dir);
321
336
}
322
337
}
323
-
EXPORT_SYMBOL(dma_direct_sync_sg_for_device);
324
338
#endif
325
339
326
340
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
327
341
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
328
342
defined(CONFIG_SWIOTLB)
329
-
void dma_direct_sync_single_for_cpu(struct device *dev,
330
-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
331
-
{
332
-
phys_addr_t paddr = dma_to_phys(dev, addr);
333
-
334
-
if (!dev_is_dma_coherent(dev)) {
335
-
arch_sync_dma_for_cpu(paddr, size, dir);
336
-
arch_sync_dma_for_cpu_all();
337
-
}
338
-
339
-
if (unlikely(is_swiotlb_buffer(paddr)))
340
-
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
341
-
}
342
-
EXPORT_SYMBOL(dma_direct_sync_single_for_cpu);
343
-
344
343
void dma_direct_sync_sg_for_cpu(struct device *dev,
345
344
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
346
345
{
···
345
376
if (!dev_is_dma_coherent(dev))
346
377
arch_sync_dma_for_cpu_all();
347
378
}
348
-
EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu);
349
-
350
-
void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
351
-
size_t size, enum dma_data_direction dir, unsigned long attrs)
352
-
{
353
-
phys_addr_t phys = dma_to_phys(dev, addr);
354
-
355
-
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
356
-
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
357
-
358
-
if (unlikely(is_swiotlb_buffer(phys)))
359
-
swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
360
-
}
361
-
EXPORT_SYMBOL(dma_direct_unmap_page);
362
379
363
380
void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
364
381
int nents, enum dma_data_direction dir, unsigned long attrs)
···
356
401
dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
357
402
attrs);
358
403
}
359
-
EXPORT_SYMBOL(dma_direct_unmap_sg);
360
404
#endif
361
-
362
-
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
363
-
unsigned long offset, size_t size, enum dma_data_direction dir,
364
-
unsigned long attrs)
365
-
{
366
-
phys_addr_t phys = page_to_phys(page) + offset;
367
-
dma_addr_t dma_addr = phys_to_dma(dev, phys);
368
-
369
-
if (unlikely(swiotlb_force == SWIOTLB_FORCE))
370
-
return swiotlb_map(dev, phys, size, dir, attrs);
371
-
372
-
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
373
-
if (swiotlb_force != SWIOTLB_NO_FORCE)
374
-
return swiotlb_map(dev, phys, size, dir, attrs);
375
-
376
-
dev_WARN_ONCE(dev, 1,
377
-
"DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
378
-
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
379
-
return DMA_MAPPING_ERROR;
380
-
}
381
-
382
-
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
383
-
arch_sync_dma_for_device(phys, size, dir);
384
-
return dma_addr;
385
-
}
386
-
EXPORT_SYMBOL(dma_direct_map_page);
387
405
388
406
int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
389
407
enum dma_data_direction dir, unsigned long attrs)
···
378
450
dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
379
451
return 0;
380
452
}
381
-
EXPORT_SYMBOL(dma_direct_map_sg);
382
453
383
454
dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
384
455
size_t size, enum dma_data_direction dir, unsigned long attrs)
···
394
467
395
468
return dma_addr;
396
469
}
397
-
EXPORT_SYMBOL(dma_direct_map_resource);
398
470
399
471
int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
400
472
void *cpu_addr, dma_addr_t dma_addr, size_t size,
+204
-10
kernel/dma/mapping.c
+204
-10
kernel/dma/mapping.c
···
105
105
}
106
106
EXPORT_SYMBOL(dmam_alloc_attrs);
107
107
108
+
static bool dma_go_direct(struct device *dev, dma_addr_t mask,
109
+
const struct dma_map_ops *ops)
110
+
{
111
+
if (likely(!ops))
112
+
return true;
113
+
#ifdef CONFIG_DMA_OPS_BYPASS
114
+
if (dev->dma_ops_bypass)
115
+
return min_not_zero(mask, dev->bus_dma_limit) >=
116
+
dma_direct_get_required_mask(dev);
117
+
#endif
118
+
return false;
119
+
}
120
+
121
+
122
+
/*
123
+
* Check if the devices uses a direct mapping for streaming DMA operations.
124
+
* This allows IOMMU drivers to set a bypass mode if the DMA mask is large
125
+
* enough.
126
+
*/
127
+
static inline bool dma_alloc_direct(struct device *dev,
128
+
const struct dma_map_ops *ops)
129
+
{
130
+
return dma_go_direct(dev, dev->coherent_dma_mask, ops);
131
+
}
132
+
133
+
static inline bool dma_map_direct(struct device *dev,
134
+
const struct dma_map_ops *ops)
135
+
{
136
+
return dma_go_direct(dev, *dev->dma_mask, ops);
137
+
}
138
+
139
+
dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
140
+
size_t offset, size_t size, enum dma_data_direction dir,
141
+
unsigned long attrs)
142
+
{
143
+
const struct dma_map_ops *ops = get_dma_ops(dev);
144
+
dma_addr_t addr;
145
+
146
+
BUG_ON(!valid_dma_direction(dir));
147
+
if (dma_map_direct(dev, ops))
148
+
addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
149
+
else
150
+
addr = ops->map_page(dev, page, offset, size, dir, attrs);
151
+
debug_dma_map_page(dev, page, offset, size, dir, addr);
152
+
153
+
return addr;
154
+
}
155
+
EXPORT_SYMBOL(dma_map_page_attrs);
156
+
157
+
void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
158
+
enum dma_data_direction dir, unsigned long attrs)
159
+
{
160
+
const struct dma_map_ops *ops = get_dma_ops(dev);
161
+
162
+
BUG_ON(!valid_dma_direction(dir));
163
+
if (dma_map_direct(dev, ops))
164
+
dma_direct_unmap_page(dev, addr, size, dir, attrs);
165
+
else if (ops->unmap_page)
166
+
ops->unmap_page(dev, addr, size, dir, attrs);
167
+
debug_dma_unmap_page(dev, addr, size, dir);
168
+
}
169
+
EXPORT_SYMBOL(dma_unmap_page_attrs);
170
+
171
+
/*
172
+
* dma_maps_sg_attrs returns 0 on error and > 0 on success.
173
+
* It should never return a value < 0.
174
+
*/
175
+
int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
176
+
enum dma_data_direction dir, unsigned long attrs)
177
+
{
178
+
const struct dma_map_ops *ops = get_dma_ops(dev);
179
+
int ents;
180
+
181
+
BUG_ON(!valid_dma_direction(dir));
182
+
if (dma_map_direct(dev, ops))
183
+
ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
184
+
else
185
+
ents = ops->map_sg(dev, sg, nents, dir, attrs);
186
+
BUG_ON(ents < 0);
187
+
debug_dma_map_sg(dev, sg, nents, ents, dir);
188
+
189
+
return ents;
190
+
}
191
+
EXPORT_SYMBOL(dma_map_sg_attrs);
192
+
193
+
void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
194
+
int nents, enum dma_data_direction dir,
195
+
unsigned long attrs)
196
+
{
197
+
const struct dma_map_ops *ops = get_dma_ops(dev);
198
+
199
+
BUG_ON(!valid_dma_direction(dir));
200
+
debug_dma_unmap_sg(dev, sg, nents, dir);
201
+
if (dma_map_direct(dev, ops))
202
+
dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
203
+
else if (ops->unmap_sg)
204
+
ops->unmap_sg(dev, sg, nents, dir, attrs);
205
+
}
206
+
EXPORT_SYMBOL(dma_unmap_sg_attrs);
207
+
208
+
dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
209
+
size_t size, enum dma_data_direction dir, unsigned long attrs)
210
+
{
211
+
const struct dma_map_ops *ops = get_dma_ops(dev);
212
+
dma_addr_t addr = DMA_MAPPING_ERROR;
213
+
214
+
BUG_ON(!valid_dma_direction(dir));
215
+
216
+
/* Don't allow RAM to be mapped */
217
+
if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
218
+
return DMA_MAPPING_ERROR;
219
+
220
+
if (dma_map_direct(dev, ops))
221
+
addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
222
+
else if (ops->map_resource)
223
+
addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
224
+
225
+
debug_dma_map_resource(dev, phys_addr, size, dir, addr);
226
+
return addr;
227
+
}
228
+
EXPORT_SYMBOL(dma_map_resource);
229
+
230
+
void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
231
+
enum dma_data_direction dir, unsigned long attrs)
232
+
{
233
+
const struct dma_map_ops *ops = get_dma_ops(dev);
234
+
235
+
BUG_ON(!valid_dma_direction(dir));
236
+
if (!dma_map_direct(dev, ops) && ops->unmap_resource)
237
+
ops->unmap_resource(dev, addr, size, dir, attrs);
238
+
debug_dma_unmap_resource(dev, addr, size, dir);
239
+
}
240
+
EXPORT_SYMBOL(dma_unmap_resource);
241
+
242
+
void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
243
+
enum dma_data_direction dir)
244
+
{
245
+
const struct dma_map_ops *ops = get_dma_ops(dev);
246
+
247
+
BUG_ON(!valid_dma_direction(dir));
248
+
if (dma_map_direct(dev, ops))
249
+
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
250
+
else if (ops->sync_single_for_cpu)
251
+
ops->sync_single_for_cpu(dev, addr, size, dir);
252
+
debug_dma_sync_single_for_cpu(dev, addr, size, dir);
253
+
}
254
+
EXPORT_SYMBOL(dma_sync_single_for_cpu);
255
+
256
+
void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
257
+
size_t size, enum dma_data_direction dir)
258
+
{
259
+
const struct dma_map_ops *ops = get_dma_ops(dev);
260
+
261
+
BUG_ON(!valid_dma_direction(dir));
262
+
if (dma_map_direct(dev, ops))
263
+
dma_direct_sync_single_for_device(dev, addr, size, dir);
264
+
else if (ops->sync_single_for_device)
265
+
ops->sync_single_for_device(dev, addr, size, dir);
266
+
debug_dma_sync_single_for_device(dev, addr, size, dir);
267
+
}
268
+
EXPORT_SYMBOL(dma_sync_single_for_device);
269
+
270
+
void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
271
+
int nelems, enum dma_data_direction dir)
272
+
{
273
+
const struct dma_map_ops *ops = get_dma_ops(dev);
274
+
275
+
BUG_ON(!valid_dma_direction(dir));
276
+
if (dma_map_direct(dev, ops))
277
+
dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
278
+
else if (ops->sync_sg_for_cpu)
279
+
ops->sync_sg_for_cpu(dev, sg, nelems, dir);
280
+
debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
281
+
}
282
+
EXPORT_SYMBOL(dma_sync_sg_for_cpu);
283
+
284
+
void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
285
+
int nelems, enum dma_data_direction dir)
286
+
{
287
+
const struct dma_map_ops *ops = get_dma_ops(dev);
288
+
289
+
BUG_ON(!valid_dma_direction(dir));
290
+
if (dma_map_direct(dev, ops))
291
+
dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
292
+
else if (ops->sync_sg_for_device)
293
+
ops->sync_sg_for_device(dev, sg, nelems, dir);
294
+
debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
295
+
}
296
+
EXPORT_SYMBOL(dma_sync_sg_for_device);
297
+
108
298
/*
109
299
* Create scatter-list for the already allocated DMA buffer.
110
300
*/
···
328
138
{
329
139
const struct dma_map_ops *ops = get_dma_ops(dev);
330
140
331
-
if (dma_is_direct(ops))
141
+
if (dma_alloc_direct(dev, ops))
332
142
return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
333
143
size, attrs);
334
144
if (!ops->get_sgtable)
···
398
208
{
399
209
const struct dma_map_ops *ops = get_dma_ops(dev);
400
210
401
-
if (dma_is_direct(ops))
211
+
if (dma_alloc_direct(dev, ops))
402
212
return dma_direct_can_mmap(dev);
403
213
return ops->mmap != NULL;
404
214
}
···
423
233
{
424
234
const struct dma_map_ops *ops = get_dma_ops(dev);
425
235
426
-
if (dma_is_direct(ops))
236
+
if (dma_alloc_direct(dev, ops))
427
237
return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
428
238
attrs);
429
239
if (!ops->mmap)
···
436
246
{
437
247
const struct dma_map_ops *ops = get_dma_ops(dev);
438
248
439
-
if (dma_is_direct(ops))
249
+
if (dma_alloc_direct(dev, ops))
440
250
return dma_direct_get_required_mask(dev);
441
251
if (ops->get_required_mask)
442
252
return ops->get_required_mask(dev);
···
467
277
/* let the implementation decide on the zone to allocate from: */
468
278
flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
469
279
470
-
if (dma_is_direct(ops))
280
+
if (dma_alloc_direct(dev, ops))
471
281
cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
472
282
else if (ops->alloc)
473
283
cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
···
499
309
return;
500
310
501
311
debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
502
-
if (dma_is_direct(ops))
312
+
if (dma_alloc_direct(dev, ops))
503
313
dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
504
314
else if (ops->free)
505
315
ops->free(dev, size, cpu_addr, dma_handle, attrs);
···
510
320
{
511
321
const struct dma_map_ops *ops = get_dma_ops(dev);
512
322
513
-
if (dma_is_direct(ops))
323
+
/*
324
+
* ->dma_supported sets the bypass flag, so we must always call
325
+
* into the method here unless the device is truly direct mapped.
326
+
*/
327
+
if (!ops)
514
328
return dma_direct_supported(dev, mask);
515
329
if (!ops->dma_supported)
516
330
return 1;
···
570
376
571
377
BUG_ON(!valid_dma_direction(dir));
572
378
573
-
if (dma_is_direct(ops))
379
+
if (dma_alloc_direct(dev, ops))
574
380
arch_dma_cache_sync(dev, vaddr, size, dir);
575
381
else if (ops->cache_sync)
576
382
ops->cache_sync(dev, vaddr, size, dir);
···
582
388
const struct dma_map_ops *ops = get_dma_ops(dev);
583
389
size_t size = SIZE_MAX;
584
390
585
-
if (dma_is_direct(ops))
391
+
if (dma_map_direct(dev, ops))
586
392
size = dma_direct_max_mapping_size(dev);
587
393
else if (ops && ops->max_mapping_size)
588
394
size = ops->max_mapping_size(dev);
···
595
401
{
596
402
const struct dma_map_ops *ops = get_dma_ops(dev);
597
403
598
-
if (dma_is_direct(ops))
404
+
if (dma_map_direct(dev, ops))
599
405
return dma_direct_need_sync(dev, dma_addr);
600
406
return ops->sync_single_for_cpu || ops->sync_single_for_device;
601
407
}