Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-6.8-2024-01-08' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping updates from Christoph Hellwig:
- reduce area lock contention for non-primary IO TLB pools (Petr
Tesarik)
- don't store redundant offsets in the dma_ranges stuctures (Robin
Murphy)
- clear dev->dma_mem when freeing per-device pools (Joakim Zhang)
* tag 'dma-mapping-6.8-2024-01-08' of git://git.infradead.org/users/hch/dma-mapping:
dma-mapping: clear dev->dma_mem to NULL after freeing it
swiotlb: reduce area lock contention for non-primary IO TLB pools
dma-mapping: don't store redundant offsets
+70
-46
-1
drivers/acpi/scan.c
-1
drivers/acpi/scan.c
-1
drivers/of/address.c
-1
drivers/of/address.c
+12
-7
include/linux/dma-direct.h
+12
-7
include/linux/dma-direct.h
···
21
21
phys_addr_t cpu_start;
22
22
dma_addr_t dma_start;
23
23
u64 size;
24
-
u64 offset;
25
24
};
26
25
27
26
static inline dma_addr_t translate_phys_to_dma(struct device *dev,
···
28
29
{
29
30
const struct bus_dma_region *m;
30
31
31
-
for (m = dev->dma_range_map; m->size; m++)
32
-
if (paddr >= m->cpu_start && paddr - m->cpu_start < m->size)
33
-
return (dma_addr_t)paddr - m->offset;
32
+
for (m = dev->dma_range_map; m->size; m++) {
33
+
u64 offset = paddr - m->cpu_start;
34
+
35
+
if (paddr >= m->cpu_start && offset < m->size)
36
+
return m->dma_start + offset;
37
+
}
34
38
35
39
/* make sure dma_capable fails when no translation is available */
36
40
return DMA_MAPPING_ERROR;
···
44
42
{
45
43
const struct bus_dma_region *m;
46
44
47
-
for (m = dev->dma_range_map; m->size; m++)
48
-
if (dma_addr >= m->dma_start && dma_addr - m->dma_start < m->size)
49
-
return (phys_addr_t)dma_addr + m->offset;
45
+
for (m = dev->dma_range_map; m->size; m++) {
46
+
u64 offset = dma_addr - m->dma_start;
47
+
48
+
if (dma_addr >= m->dma_start && offset < m->size)
49
+
return m->cpu_start + offset;
50
+
}
50
51
51
52
return (phys_addr_t)-1;
52
53
}
+3
-1
kernel/dma/coherent.c
+3
-1
kernel/dma/coherent.c
-1
kernel/dma/direct.c
-1
kernel/dma/direct.c
+55
-35
kernel/dma/swiotlb.c
+55
-35
kernel/dma/swiotlb.c
···
957
957
#endif /* CONFIG_DEBUG_FS */
958
958
959
959
/**
960
-
* swiotlb_area_find_slots() - search for slots in one IO TLB memory area
960
+
* swiotlb_search_pool_area() - search one memory area in one pool
961
961
* @dev: Device which maps the buffer.
962
962
* @pool: Memory pool to be searched.
963
963
* @area_index: Index of the IO TLB memory area to be searched.
···
972
972
*
973
973
* Return: Index of the first allocated slot, or -1 on error.
974
974
*/
975
-
static int swiotlb_area_find_slots(struct device *dev, struct io_tlb_pool *pool,
975
+
static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
976
976
int area_index, phys_addr_t orig_addr, size_t alloc_size,
977
977
unsigned int alloc_align_mask)
978
978
{
···
1066
1066
return slot_index;
1067
1067
}
1068
1068
1069
+
#ifdef CONFIG_SWIOTLB_DYNAMIC
1070
+
1069
1071
/**
1070
-
* swiotlb_pool_find_slots() - search for slots in one memory pool
1072
+
* swiotlb_search_area() - search one memory area in all pools
1071
1073
* @dev: Device which maps the buffer.
1072
-
* @pool: Memory pool to be searched.
1074
+
* @start_cpu: Start CPU number.
1075
+
* @cpu_offset: Offset from @start_cpu.
1073
1076
* @orig_addr: Original (non-bounced) IO buffer address.
1074
1077
* @alloc_size: Total requested size of the bounce buffer,
1075
1078
* including initial alignment padding.
1076
1079
* @alloc_align_mask: Required alignment of the allocated buffer.
1080
+
* @retpool: Used memory pool, updated on return.
1077
1081
*
1078
-
* Search through one memory pool to find a sequence of slots that match the
1082
+
* Search one memory area in all pools for a sequence of slots that match the
1079
1083
* allocation constraints.
1080
1084
*
1081
1085
* Return: Index of the first allocated slot, or -1 on error.
1082
1086
*/
1083
-
static int swiotlb_pool_find_slots(struct device *dev, struct io_tlb_pool *pool,
1084
-
phys_addr_t orig_addr, size_t alloc_size,
1085
-
unsigned int alloc_align_mask)
1087
+
static int swiotlb_search_area(struct device *dev, int start_cpu,
1088
+
int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1089
+
unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1086
1090
{
1087
-
int start = raw_smp_processor_id() & (pool->nareas - 1);
1088
-
int i = start, index;
1091
+
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1092
+
struct io_tlb_pool *pool;
1093
+
int area_index;
1094
+
int index = -1;
1089
1095
1090
-
do {
1091
-
index = swiotlb_area_find_slots(dev, pool, i, orig_addr,
1092
-
alloc_size, alloc_align_mask);
1093
-
if (index >= 0)
1094
-
return index;
1095
-
if (++i >= pool->nareas)
1096
-
i = 0;
1097
-
} while (i != start);
1098
-
1099
-
return -1;
1096
+
rcu_read_lock();
1097
+
list_for_each_entry_rcu(pool, &mem->pools, node) {
1098
+
if (cpu_offset >= pool->nareas)
1099
+
continue;
1100
+
area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1101
+
index = swiotlb_search_pool_area(dev, pool, area_index,
1102
+
orig_addr, alloc_size,
1103
+
alloc_align_mask);
1104
+
if (index >= 0) {
1105
+
*retpool = pool;
1106
+
break;
1107
+
}
1108
+
}
1109
+
rcu_read_unlock();
1110
+
return index;
1100
1111
}
1101
-
1102
-
#ifdef CONFIG_SWIOTLB_DYNAMIC
1103
1112
1104
1113
/**
1105
1114
* swiotlb_find_slots() - search for slots in the whole swiotlb
···
1133
1124
unsigned long nslabs;
1134
1125
unsigned long flags;
1135
1126
u64 phys_limit;
1127
+
int cpu, i;
1136
1128
int index;
1137
1129
1138
-
rcu_read_lock();
1139
-
list_for_each_entry_rcu(pool, &mem->pools, node) {
1140
-
index = swiotlb_pool_find_slots(dev, pool, orig_addr,
1141
-
alloc_size, alloc_align_mask);
1142
-
if (index >= 0) {
1143
-
rcu_read_unlock();
1130
+
cpu = raw_smp_processor_id();
1131
+
for (i = 0; i < default_nareas; ++i) {
1132
+
index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1133
+
alloc_align_mask, &pool);
1134
+
if (index >= 0)
1144
1135
goto found;
1145
-
}
1146
1136
}
1147
-
rcu_read_unlock();
1137
+
1148
1138
if (!mem->can_grow)
1149
1139
return -1;
1150
1140
···
1156
1148
if (!pool)
1157
1149
return -1;
1158
1150
1159
-
index = swiotlb_pool_find_slots(dev, pool, orig_addr,
1160
-
alloc_size, alloc_align_mask);
1151
+
index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1152
+
alloc_size, alloc_align_mask);
1161
1153
if (index < 0) {
1162
1154
swiotlb_dyn_free(&pool->rcu);
1163
1155
return -1;
···
1200
1192
size_t alloc_size, unsigned int alloc_align_mask,
1201
1193
struct io_tlb_pool **retpool)
1202
1194
{
1203
-
*retpool = &dev->dma_io_tlb_mem->defpool;
1204
-
return swiotlb_pool_find_slots(dev, *retpool,
1205
-
orig_addr, alloc_size, alloc_align_mask);
1195
+
struct io_tlb_pool *pool;
1196
+
int start, i;
1197
+
int index;
1198
+
1199
+
*retpool = pool = &dev->dma_io_tlb_mem->defpool;
1200
+
i = start = raw_smp_processor_id() & (pool->nareas - 1);
1201
+
do {
1202
+
index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1203
+
alloc_size, alloc_align_mask);
1204
+
if (index >= 0)
1205
+
return index;
1206
+
if (++i >= pool->nareas)
1207
+
i = 0;
1208
+
} while (i != start);
1209
+
return -1;
1206
1210
}
1207
1211
1208
1212
#endif /* CONFIG_SWIOTLB_DYNAMIC */