Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
swiotlb: merge swiotlb-xen initialization into swiotlb
Reuse the generic swiotlb initialization for xen-swiotlb. For ARM/ARM64
this works trivially, while for x86 xen_swiotlb_fixup needs to be passed
as the remap argument to swiotlb_init_remap/swiotlb_init_late.
Note that the lower bound of the swiotlb size is changed to the smaller
IO_TLB_MIN_SLABS based value with this patch, but that is fine as the
2MB value used in Xen before was just an optimization and is not the
hard lower bound.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Tested-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
+28
-155
+11
-10
arch/arm/xen/mm.c
+11
-10
arch/arm/xen/mm.c
···
23
23
#include <asm/xen/hypercall.h>
24
24
#include <asm/xen/interface.h>
25
25
26
-
unsigned long xen_get_swiotlb_free_pages(unsigned int order)
26
+
static gfp_t xen_swiotlb_gfp(void)
27
27
{
28
28
phys_addr_t base;
29
-
gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM;
30
29
u64 i;
31
30
32
31
for_each_mem_range(i, &base, NULL) {
33
32
if (base < (phys_addr_t)0xffffffff) {
34
33
if (IS_ENABLED(CONFIG_ZONE_DMA32))
35
-
flags |= __GFP_DMA32;
36
-
else
37
-
flags |= __GFP_DMA;
38
-
break;
34
+
return __GFP_DMA32;
35
+
return __GFP_DMA;
39
36
}
40
37
}
41
-
return __get_free_pages(flags, order);
38
+
39
+
return GFP_KERNEL;
42
40
}
43
41
44
42
static bool hypercall_cflush = false;
···
138
140
if (!xen_swiotlb_detect())
139
141
return 0;
140
142
141
-
rc = xen_swiotlb_init();
142
143
/* we can work with the default swiotlb */
143
-
if (rc < 0 && rc != -EEXIST)
144
-
return rc;
144
+
if (!io_tlb_default_mem.nslabs) {
145
+
rc = swiotlb_init_late(swiotlb_size_or_default(),
146
+
xen_swiotlb_gfp(), NULL);
147
+
if (rc < 0)
148
+
return rc;
149
+
}
145
150
146
151
cflush.op = 0;
147
152
cflush.a.dev_bus_addr = 0;
-5
arch/x86/include/asm/xen/page.h
-5
arch/x86/include/asm/xen/page.h
+10
-10
arch/x86/kernel/pci-dma.c
+10
-10
arch/x86/kernel/pci-dma.c
···
72
72
#endif /* CONFIG_SWIOTLB */
73
73
74
74
#ifdef CONFIG_SWIOTLB_XEN
75
-
static bool xen_swiotlb;
76
-
77
75
static void __init pci_xen_swiotlb_init(void)
78
76
{
79
77
if (!xen_initial_domain() && !x86_swiotlb_enable)
80
78
return;
81
79
x86_swiotlb_enable = true;
82
-
xen_swiotlb = true;
83
-
xen_swiotlb_init_early();
80
+
x86_swiotlb_flags |= SWIOTLB_ANY;
81
+
swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup);
84
82
dma_ops = &xen_swiotlb_dma_ops;
85
83
if (IS_ENABLED(CONFIG_PCI))
86
84
pci_request_acs();
···
86
88
87
89
int pci_xen_swiotlb_init_late(void)
88
90
{
89
-
int rc;
90
-
91
-
if (xen_swiotlb)
91
+
if (dma_ops == &xen_swiotlb_dma_ops)
92
92
return 0;
93
93
94
-
rc = xen_swiotlb_init();
95
-
if (rc)
96
-
return rc;
94
+
/* we can work with the default swiotlb */
95
+
if (!io_tlb_default_mem.nslabs) {
96
+
int rc = swiotlb_init_late(swiotlb_size_or_default(),
97
+
GFP_KERNEL, xen_swiotlb_fixup);
98
+
if (rc < 0)
99
+
return rc;
100
+
}
97
101
98
102
/* XXX: this switches the dma ops under live devices! */
99
103
dma_ops = &xen_swiotlb_dma_ops;
+1
-127
drivers/xen/swiotlb-xen.c
+1
-127
drivers/xen/swiotlb-xen.c
···
104
104
return 0;
105
105
}
106
106
107
-
static int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
107
+
int xen_swiotlb_fixup(void *buf, unsigned long nslabs)
108
108
{
109
109
int rc;
110
110
unsigned int order = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT);
···
129
129
} while (i < nslabs);
130
130
return 0;
131
131
}
132
-
133
-
enum xen_swiotlb_err {
134
-
XEN_SWIOTLB_UNKNOWN = 0,
135
-
XEN_SWIOTLB_ENOMEM,
136
-
XEN_SWIOTLB_EFIXUP
137
-
};
138
-
139
-
static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
140
-
{
141
-
switch (err) {
142
-
case XEN_SWIOTLB_ENOMEM:
143
-
return "Cannot allocate Xen-SWIOTLB buffer\n";
144
-
case XEN_SWIOTLB_EFIXUP:
145
-
return "Failed to get contiguous memory for DMA from Xen!\n"\
146
-
"You either: don't have the permissions, do not have"\
147
-
" enough free memory under 4GB, or the hypervisor memory"\
148
-
" is too fragmented!";
149
-
default:
150
-
break;
151
-
}
152
-
return "";
153
-
}
154
-
155
-
int xen_swiotlb_init(void)
156
-
{
157
-
enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
158
-
unsigned long bytes = swiotlb_size_or_default();
159
-
unsigned long nslabs = bytes >> IO_TLB_SHIFT;
160
-
unsigned int order, repeat = 3;
161
-
int rc = -ENOMEM;
162
-
char *start;
163
-
164
-
if (io_tlb_default_mem.nslabs) {
165
-
pr_warn("swiotlb buffer already initialized\n");
166
-
return -EEXIST;
167
-
}
168
-
169
-
retry:
170
-
m_ret = XEN_SWIOTLB_ENOMEM;
171
-
order = get_order(bytes);
172
-
173
-
/*
174
-
* Get IO TLB memory from any location.
175
-
*/
176
-
#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
177
-
#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
178
-
while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
179
-
start = (void *)xen_get_swiotlb_free_pages(order);
180
-
if (start)
181
-
break;
182
-
order--;
183
-
}
184
-
if (!start)
185
-
goto exit;
186
-
if (order != get_order(bytes)) {
187
-
pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n",
188
-
(PAGE_SIZE << order) >> 20);
189
-
nslabs = SLABS_PER_PAGE << order;
190
-
bytes = nslabs << IO_TLB_SHIFT;
191
-
}
192
-
193
-
/*
194
-
* And replace that memory with pages under 4GB.
195
-
*/
196
-
rc = xen_swiotlb_fixup(start, nslabs);
197
-
if (rc) {
198
-
free_pages((unsigned long)start, order);
199
-
m_ret = XEN_SWIOTLB_EFIXUP;
200
-
goto error;
201
-
}
202
-
rc = swiotlb_late_init_with_tbl(start, nslabs);
203
-
if (rc)
204
-
return rc;
205
-
return 0;
206
-
error:
207
-
if (nslabs > 1024 && repeat--) {
208
-
/* Min is 2MB */
209
-
nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
210
-
bytes = nslabs << IO_TLB_SHIFT;
211
-
pr_info("Lowering to %luMB\n", bytes >> 20);
212
-
goto retry;
213
-
}
214
-
exit:
215
-
pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc);
216
-
return rc;
217
-
}
218
-
219
-
#ifdef CONFIG_X86
220
-
void __init xen_swiotlb_init_early(void)
221
-
{
222
-
unsigned long bytes = swiotlb_size_or_default();
223
-
unsigned long nslabs = bytes >> IO_TLB_SHIFT;
224
-
unsigned int repeat = 3;
225
-
char *start;
226
-
int rc;
227
-
228
-
retry:
229
-
/*
230
-
* Get IO TLB memory from any location.
231
-
*/
232
-
start = memblock_alloc(PAGE_ALIGN(bytes),
233
-
IO_TLB_SEGSIZE << IO_TLB_SHIFT);
234
-
if (!start)
235
-
panic("%s: Failed to allocate %lu bytes\n",
236
-
__func__, PAGE_ALIGN(bytes));
237
-
238
-
/*
239
-
* And replace that memory with pages under 4GB.
240
-
*/
241
-
rc = xen_swiotlb_fixup(start, nslabs);
242
-
if (rc) {
243
-
memblock_free(start, PAGE_ALIGN(bytes));
244
-
if (nslabs > 1024 && repeat--) {
245
-
/* Min is 2MB */
246
-
nslabs = max(1024UL, ALIGN(nslabs >> 1, IO_TLB_SEGSIZE));
247
-
bytes = nslabs << IO_TLB_SHIFT;
248
-
pr_info("Lowering to %luMB\n", bytes >> 20);
249
-
goto retry;
250
-
}
251
-
panic("%s (rc:%d)", xen_swiotlb_error(XEN_SWIOTLB_EFIXUP), rc);
252
-
}
253
-
254
-
if (swiotlb_init_with_tbl(start, nslabs, SWIOTLB_VERBOSE))
255
-
panic("Cannot allocate SWIOTLB buffer");
256
-
}
257
-
#endif /* CONFIG_X86 */
258
132
259
133
static void *
260
134
xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
-1
include/xen/arm/page.h
-1
include/xen/arm/page.h
+6
-2
include/xen/swiotlb-xen.h
+6
-2
include/xen/swiotlb-xen.h
···
10
10
void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle,
11
11
size_t size, enum dma_data_direction dir);
12
12
13
-
int xen_swiotlb_init(void);
14
-
void __init xen_swiotlb_init_early(void);
13
+
#ifdef CONFIG_SWIOTLB_XEN
14
+
int xen_swiotlb_fixup(void *buf, unsigned long nslabs);
15
+
#else
16
+
#define xen_swiotlb_fixup NULL
17
+
#endif
18
+
15
19
extern const struct dma_map_ops xen_swiotlb_dma_ops;
16
20
17
21
#endif /* __LINUX_SWIOTLB_XEN_H */