Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping updates from Christoph Hellwig:
- reduce duplicate swiotlb pool lookups (Michael Kelley)
- minor small fixes (Yicong Yang, Yang Li)
* tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping:
swiotlb: fix kernel-doc description for swiotlb_del_transient
swiotlb: reduce swiotlb pool lookups
dma-mapping: benchmark: Don't starve others when doing the test
+146
-106
+4
-7
drivers/iommu/dma-iommu.c
+4
-7
drivers/iommu/dma-iommu.c
···
1078
1078
if (!dev_is_dma_coherent(dev))
1079
1079
arch_sync_dma_for_cpu(phys, size, dir);
1080
1080
1081
-
if (is_swiotlb_buffer(dev, phys))
1082
-
swiotlb_sync_single_for_cpu(dev, phys, size, dir);
1081
+
swiotlb_sync_single_for_cpu(dev, phys, size, dir);
1083
1082
}
1084
1083
1085
1084
static void iommu_dma_sync_single_for_device(struct device *dev,
···
1090
1091
return;
1091
1092
1092
1093
phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
1093
-
if (is_swiotlb_buffer(dev, phys))
1094
-
swiotlb_sync_single_for_device(dev, phys, size, dir);
1094
+
swiotlb_sync_single_for_device(dev, phys, size, dir);
1095
1095
1096
1096
if (!dev_is_dma_coherent(dev))
1097
1097
arch_sync_dma_for_device(phys, size, dir);
···
1184
1186
arch_sync_dma_for_device(phys, size, dir);
1185
1187
1186
1188
iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
1187
-
if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
1189
+
if (iova == DMA_MAPPING_ERROR)
1188
1190
swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1189
1191
return iova;
1190
1192
}
···
1204
1206
1205
1207
__iommu_dma_unmap(dev, dma_handle, size);
1206
1208
1207
-
if (unlikely(is_swiotlb_buffer(dev, phys)))
1208
-
swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1209
+
swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1209
1210
}
1210
1211
1211
1212
/*
+20
-11
drivers/xen/swiotlb-xen.c
+20
-11
drivers/xen/swiotlb-xen.c
···
88
88
return 0;
89
89
}
90
90
91
-
static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
91
+
static struct io_tlb_pool *xen_swiotlb_find_pool(struct device *dev,
92
+
dma_addr_t dma_addr)
92
93
{
93
94
unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
94
95
unsigned long xen_pfn = bfn_to_local_pfn(bfn);
···
100
99
* in our domain. Therefore _only_ check address within our domain.
101
100
*/
102
101
if (pfn_valid(PFN_DOWN(paddr)))
103
-
return is_swiotlb_buffer(dev, paddr);
104
-
return 0;
102
+
return swiotlb_find_pool(dev, paddr);
103
+
return NULL;
105
104
}
106
105
107
106
#ifdef CONFIG_X86
···
228
227
* Ensure that the address returned is DMA'ble
229
228
*/
230
229
if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
231
-
swiotlb_tbl_unmap_single(dev, map, size, dir,
232
-
attrs | DMA_ATTR_SKIP_CPU_SYNC);
230
+
__swiotlb_tbl_unmap_single(dev, map, size, dir,
231
+
attrs | DMA_ATTR_SKIP_CPU_SYNC,
232
+
swiotlb_find_pool(dev, map));
233
233
return DMA_MAPPING_ERROR;
234
234
}
235
235
···
256
254
size_t size, enum dma_data_direction dir, unsigned long attrs)
257
255
{
258
256
phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);
257
+
struct io_tlb_pool *pool;
259
258
260
259
BUG_ON(dir == DMA_NONE);
261
260
···
268
265
}
269
266
270
267
/* NOTE: We use dev_addr here, not paddr! */
271
-
if (is_xen_swiotlb_buffer(hwdev, dev_addr))
272
-
swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
268
+
pool = xen_swiotlb_find_pool(hwdev, dev_addr);
269
+
if (pool)
270
+
__swiotlb_tbl_unmap_single(hwdev, paddr, size, dir,
271
+
attrs, pool);
273
272
}
274
273
275
274
static void
···
279
274
size_t size, enum dma_data_direction dir)
280
275
{
281
276
phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
277
+
struct io_tlb_pool *pool;
282
278
283
279
if (!dev_is_dma_coherent(dev)) {
284
280
if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
···
288
282
xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
289
283
}
290
284
291
-
if (is_xen_swiotlb_buffer(dev, dma_addr))
292
-
swiotlb_sync_single_for_cpu(dev, paddr, size, dir);
285
+
pool = xen_swiotlb_find_pool(dev, dma_addr);
286
+
if (pool)
287
+
__swiotlb_sync_single_for_cpu(dev, paddr, size, dir, pool);
293
288
}
294
289
295
290
static void
···
298
291
size_t size, enum dma_data_direction dir)
299
292
{
300
293
phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
294
+
struct io_tlb_pool *pool;
301
295
302
-
if (is_xen_swiotlb_buffer(dev, dma_addr))
303
-
swiotlb_sync_single_for_device(dev, paddr, size, dir);
296
+
pool = xen_swiotlb_find_pool(dev, dma_addr);
297
+
if (pool)
298
+
__swiotlb_sync_single_for_device(dev, paddr, size, dir, pool);
304
299
305
300
if (!dev_is_dma_coherent(dev)) {
306
301
if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
+1
-1
include/linux/scatterlist.h
+1
-1
include/linux/scatterlist.h
···
332
332
* Description:
333
333
* Returns true if the scatterlist was marked for SWIOTLB bouncing. Not all
334
334
* elements may have been bounced, so the caller would have to check
335
-
* individual SG entries with is_swiotlb_buffer().
335
+
* individual SG entries with swiotlb_find_pool().
336
336
*/
337
337
static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
338
338
{
+62
-43
include/linux/swiotlb.h
+62
-43
include/linux/swiotlb.h
···
42
42
int (*remap)(void *tlb, unsigned long nslabs));
43
43
extern void __init swiotlb_update_mem_attributes(void);
44
44
45
-
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
46
-
size_t mapping_size,
47
-
unsigned int alloc_aligned_mask, enum dma_data_direction dir,
48
-
unsigned long attrs);
49
-
50
-
extern void swiotlb_tbl_unmap_single(struct device *hwdev,
51
-
phys_addr_t tlb_addr,
52
-
size_t mapping_size,
53
-
enum dma_data_direction dir,
54
-
unsigned long attrs);
55
-
56
-
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
57
-
size_t size, enum dma_data_direction dir);
58
-
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
59
-
size_t size, enum dma_data_direction dir);
60
-
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
61
-
size_t size, enum dma_data_direction dir, unsigned long attrs);
62
-
63
45
#ifdef CONFIG_SWIOTLB
64
46
65
47
/**
···
125
143
#endif
126
144
};
127
145
128
-
#ifdef CONFIG_SWIOTLB_DYNAMIC
129
-
130
-
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
131
-
132
-
#else
133
-
134
-
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
135
-
phys_addr_t paddr)
136
-
{
137
-
return &dev->dma_io_tlb_mem->defpool;
138
-
}
139
-
140
-
#endif
146
+
struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
141
147
142
148
/**
143
-
* is_swiotlb_buffer() - check if a physical address belongs to a swiotlb
149
+
* swiotlb_find_pool() - find swiotlb pool to which a physical address belongs
144
150
* @dev: Device which has mapped the buffer.
145
151
* @paddr: Physical address within the DMA buffer.
146
152
*
147
-
* Check if @paddr points into a bounce buffer.
153
+
* Find the swiotlb pool that @paddr points into.
148
154
*
149
155
* Return:
150
-
* * %true if @paddr points into a bounce buffer
151
-
* * %false otherwise
156
+
* * pool address if @paddr points into a bounce buffer
157
+
* * NULL if @paddr does not point into a bounce buffer. As such, this function
158
+
* can be used to determine if @paddr denotes a swiotlb bounce buffer.
152
159
*/
153
-
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
160
+
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
161
+
phys_addr_t paddr)
154
162
{
155
163
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
156
164
157
165
if (!mem)
158
-
return false;
166
+
return NULL;
159
167
160
168
#ifdef CONFIG_SWIOTLB_DYNAMIC
161
169
/*
···
154
182
* If a SWIOTLB address is checked on another CPU, then it was
155
183
* presumably loaded by the device driver from an unspecified private
156
184
* data structure. Make sure that this load is ordered before reading
157
-
* dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().
185
+
* dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool().
158
186
*
159
187
* This barrier pairs with smp_mb() in swiotlb_find_slots().
160
188
*/
161
189
smp_rmb();
162
-
return READ_ONCE(dev->dma_uses_io_tlb) &&
163
-
swiotlb_find_pool(dev, paddr);
190
+
if (READ_ONCE(dev->dma_uses_io_tlb))
191
+
return __swiotlb_find_pool(dev, paddr);
164
192
#else
165
-
return paddr >= mem->defpool.start && paddr < mem->defpool.end;
193
+
if (paddr >= mem->defpool.start && paddr < mem->defpool.end)
194
+
return &mem->defpool;
166
195
#endif
196
+
197
+
return NULL;
167
198
}
168
199
169
200
static inline bool is_swiotlb_force_bounce(struct device *dev)
···
194
219
{
195
220
}
196
221
197
-
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
222
+
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
223
+
phys_addr_t paddr)
198
224
{
199
-
return false;
225
+
return NULL;
200
226
}
201
227
static inline bool is_swiotlb_force_bounce(struct device *dev)
202
228
{
···
235
259
return 0;
236
260
}
237
261
#endif /* CONFIG_SWIOTLB */
262
+
263
+
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
264
+
size_t mapping_size, unsigned int alloc_aligned_mask,
265
+
enum dma_data_direction dir, unsigned long attrs);
266
+
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
267
+
size_t size, enum dma_data_direction dir, unsigned long attrs);
268
+
269
+
void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
270
+
size_t mapping_size, enum dma_data_direction dir,
271
+
unsigned long attrs, struct io_tlb_pool *pool);
272
+
static inline void swiotlb_tbl_unmap_single(struct device *dev,
273
+
phys_addr_t addr, size_t size, enum dma_data_direction dir,
274
+
unsigned long attrs)
275
+
{
276
+
struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
277
+
278
+
if (unlikely(pool))
279
+
__swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);
280
+
}
281
+
282
+
void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
283
+
size_t size, enum dma_data_direction dir,
284
+
struct io_tlb_pool *pool);
285
+
static inline void swiotlb_sync_single_for_device(struct device *dev,
286
+
phys_addr_t addr, size_t size, enum dma_data_direction dir)
287
+
{
288
+
struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
289
+
290
+
if (unlikely(pool))
291
+
__swiotlb_sync_single_for_device(dev, addr, size, dir, pool);
292
+
}
293
+
294
+
void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
295
+
size_t size, enum dma_data_direction dir,
296
+
struct io_tlb_pool *pool);
297
+
static inline void swiotlb_sync_single_for_cpu(struct device *dev,
298
+
phys_addr_t addr, size_t size, enum dma_data_direction dir)
299
+
{
300
+
struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
301
+
302
+
if (unlikely(pool))
303
+
__swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);
304
+
}
238
305
239
306
extern void swiotlb_print_info(void);
240
307
+3
-7
kernel/dma/direct.c
+3
-7
kernel/dma/direct.c
···
404
404
for_each_sg(sgl, sg, nents, i) {
405
405
phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
406
406
407
-
if (unlikely(is_swiotlb_buffer(dev, paddr)))
408
-
swiotlb_sync_single_for_device(dev, paddr, sg->length,
409
-
dir);
407
+
swiotlb_sync_single_for_device(dev, paddr, sg->length, dir);
410
408
411
409
if (!dev_is_dma_coherent(dev))
412
410
arch_sync_dma_for_device(paddr, sg->length,
···
428
430
if (!dev_is_dma_coherent(dev))
429
431
arch_sync_dma_for_cpu(paddr, sg->length, dir);
430
432
431
-
if (unlikely(is_swiotlb_buffer(dev, paddr)))
432
-
swiotlb_sync_single_for_cpu(dev, paddr, sg->length,
433
-
dir);
433
+
swiotlb_sync_single_for_cpu(dev, paddr, sg->length, dir);
434
434
435
435
if (dir == DMA_FROM_DEVICE)
436
436
arch_dma_mark_clean(paddr, sg->length);
···
636
640
bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr)
637
641
{
638
642
return !dev_is_dma_coherent(dev) ||
639
-
is_swiotlb_buffer(dev, dma_to_phys(dev, dma_addr));
643
+
swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr));
640
644
}
641
645
642
646
/**
+3
-6
kernel/dma/direct.h
+3
-6
kernel/dma/direct.h
···
58
58
{
59
59
phys_addr_t paddr = dma_to_phys(dev, addr);
60
60
61
-
if (unlikely(is_swiotlb_buffer(dev, paddr)))
62
-
swiotlb_sync_single_for_device(dev, paddr, size, dir);
61
+
swiotlb_sync_single_for_device(dev, paddr, size, dir);
63
62
64
63
if (!dev_is_dma_coherent(dev))
65
64
arch_sync_dma_for_device(paddr, size, dir);
···
74
75
arch_sync_dma_for_cpu_all();
75
76
}
76
77
77
-
if (unlikely(is_swiotlb_buffer(dev, paddr)))
78
-
swiotlb_sync_single_for_cpu(dev, paddr, size, dir);
78
+
swiotlb_sync_single_for_cpu(dev, paddr, size, dir);
79
79
80
80
if (dir == DMA_FROM_DEVICE)
81
81
arch_dma_mark_clean(paddr, size);
···
119
121
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
120
122
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
121
123
122
-
if (unlikely(is_swiotlb_buffer(dev, phys)))
123
-
swiotlb_tbl_unmap_single(dev, phys, size, dir,
124
+
swiotlb_tbl_unmap_single(dev, phys, size, dir,
124
125
attrs | DMA_ATTR_SKIP_CPU_SYNC);
125
126
}
126
127
#endif /* _KERNEL_DMA_DIRECT_H */
+16
kernel/dma/map_benchmark.c
+16
kernel/dma/map_benchmark.c
···
89
89
atomic64_add(map_sq, &map->sum_sq_map);
90
90
atomic64_add(unmap_sq, &map->sum_sq_unmap);
91
91
atomic64_inc(&map->loops);
92
+
93
+
/*
94
+
* We may test for a long time so periodically check whether
95
+
* we need to schedule to avoid starving the others. Otherwise
96
+
* we may hangup the kernel in a non-preemptible kernel when
97
+
* the test kthreads number >= CPU number, the test kthreads
98
+
* will run endless on every CPU since the thread resposible
99
+
* for notifying the kthread stop (in do_map_benchmark())
100
+
* could not be scheduled.
101
+
*
102
+
* Note this may degrade the test concurrency since the test
103
+
* threads may need to share the CPU time with other load
104
+
* in the system. So it's recommended to run this benchmark
105
+
* on an idle system.
106
+
*/
107
+
cond_resched();
92
108
}
93
109
94
110
out:
+37
-31
kernel/dma/swiotlb.c
+37
-31
kernel/dma/swiotlb.c
···
763
763
}
764
764
765
765
/**
766
-
* swiotlb_find_pool() - find the IO TLB pool for a physical address
766
+
* __swiotlb_find_pool() - find the IO TLB pool for a physical address
767
767
* @dev: Device which has mapped the DMA buffer.
768
768
* @paddr: Physical address within the DMA buffer.
769
769
*
770
770
* Find the IO TLB memory pool descriptor which contains the given physical
771
-
* address, if any.
771
+
* address, if any. This function is for use only when the dev is known to
772
+
* be using swiotlb. Use swiotlb_find_pool() for the more general case
773
+
* when this condition is not met.
772
774
*
773
775
* Return: Memory pool which contains @paddr, or %NULL if none.
774
776
*/
775
-
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
777
+
struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
776
778
{
777
779
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
778
780
struct io_tlb_pool *pool;
···
857
855
* Bounce: copy the swiotlb buffer from or back to the original dma location
858
856
*/
859
857
static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
860
-
enum dma_data_direction dir)
858
+
enum dma_data_direction dir, struct io_tlb_pool *mem)
861
859
{
862
-
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
863
860
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
864
861
phys_addr_t orig_addr = mem->slots[index].orig_addr;
865
862
size_t alloc_size = mem->slots[index].alloc_size;
···
1244
1243
* that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1245
1244
* atomicity).
1246
1245
*
1247
-
* See also the comment in is_swiotlb_buffer().
1246
+
* See also the comment in swiotlb_find_pool().
1248
1247
*/
1249
1248
smp_mb();
1250
1249
···
1436
1435
* hardware behavior. Use of swiotlb is supposed to be transparent,
1437
1436
* i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1438
1437
*/
1439
-
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
1438
+
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);
1440
1439
return tlb_addr;
1441
1440
}
1442
1441
1443
-
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
1442
+
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1443
+
struct io_tlb_pool *mem)
1444
1444
{
1445
-
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
1446
1445
unsigned long flags;
1447
1446
unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1448
1447
int index, nslots, aindex;
···
1500
1499
* swiotlb_del_transient() - delete a transient memory pool
1501
1500
* @dev: Device which mapped the buffer.
1502
1501
* @tlb_addr: Physical address within a bounce buffer.
1502
+
* @pool: Pointer to the transient memory pool to be checked and deleted.
1503
1503
*
1504
1504
* Check whether the address belongs to a transient SWIOTLB memory pool.
1505
1505
* If yes, then delete the pool.
1506
1506
*
1507
1507
* Return: %true if @tlb_addr belonged to a transient pool that was released.
1508
1508
*/
1509
-
static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
1509
+
static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1510
+
struct io_tlb_pool *pool)
1510
1511
{
1511
-
struct io_tlb_pool *pool;
1512
-
1513
-
pool = swiotlb_find_pool(dev, tlb_addr);
1514
1512
if (!pool->transient)
1515
1513
return false;
1516
1514
···
1522
1522
#else /* !CONFIG_SWIOTLB_DYNAMIC */
1523
1523
1524
1524
static inline bool swiotlb_del_transient(struct device *dev,
1525
-
phys_addr_t tlb_addr)
1525
+
phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1526
1526
{
1527
1527
return false;
1528
1528
}
···
1532
1532
/*
1533
1533
* tlb_addr is the physical address of the bounce buffer to unmap.
1534
1534
*/
1535
-
void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1536
-
size_t mapping_size, enum dma_data_direction dir,
1537
-
unsigned long attrs)
1535
+
void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1536
+
size_t mapping_size, enum dma_data_direction dir,
1537
+
unsigned long attrs, struct io_tlb_pool *pool)
1538
1538
{
1539
1539
/*
1540
1540
* First, sync the memory before unmapping the entry
1541
1541
*/
1542
1542
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1543
1543
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1544
-
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
1544
+
swiotlb_bounce(dev, tlb_addr, mapping_size,
1545
+
DMA_FROM_DEVICE, pool);
1545
1546
1546
-
if (swiotlb_del_transient(dev, tlb_addr))
1547
+
if (swiotlb_del_transient(dev, tlb_addr, pool))
1547
1548
return;
1548
-
swiotlb_release_slots(dev, tlb_addr);
1549
+
swiotlb_release_slots(dev, tlb_addr, pool);
1549
1550
}
1550
1551
1551
-
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1552
-
size_t size, enum dma_data_direction dir)
1552
+
void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1553
+
size_t size, enum dma_data_direction dir,
1554
+
struct io_tlb_pool *pool)
1553
1555
{
1554
1556
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1555
-
swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE);
1557
+
swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);
1556
1558
else
1557
1559
BUG_ON(dir != DMA_FROM_DEVICE);
1558
1560
}
1559
1561
1560
-
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1561
-
size_t size, enum dma_data_direction dir)
1562
+
void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1563
+
size_t size, enum dma_data_direction dir,
1564
+
struct io_tlb_pool *pool)
1562
1565
{
1563
1566
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1564
-
swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE);
1567
+
swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);
1565
1568
else
1566
1569
BUG_ON(dir != DMA_TO_DEVICE);
1567
1570
}
···
1588
1585
/* Ensure that the address returned is DMA'ble */
1589
1586
dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1590
1587
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1591
-
swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1592
-
attrs | DMA_ATTR_SKIP_CPU_SYNC);
1588
+
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1589
+
attrs | DMA_ATTR_SKIP_CPU_SYNC,
1590
+
swiotlb_find_pool(dev, swiotlb_addr));
1593
1591
dev_WARN_ONCE(dev, 1,
1594
1592
"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1595
1593
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
···
1768
1764
if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1769
1765
dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1770
1766
&tlb_addr);
1771
-
swiotlb_release_slots(dev, tlb_addr);
1767
+
swiotlb_release_slots(dev, tlb_addr, pool);
1772
1768
return NULL;
1773
1769
}
1774
1770
···
1778
1774
bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1779
1775
{
1780
1776
phys_addr_t tlb_addr = page_to_phys(page);
1777
+
struct io_tlb_pool *pool;
1781
1778
1782
-
if (!is_swiotlb_buffer(dev, tlb_addr))
1779
+
pool = swiotlb_find_pool(dev, tlb_addr);
1780
+
if (!pool)
1783
1781
return false;
1784
1782
1785
-
swiotlb_release_slots(dev, tlb_addr);
1783
+
swiotlb_release_slots(dev, tlb_addr, pool);
1786
1784
1787
1785
return true;
1788
1786
}