Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-5.0-2' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping fix from Christoph Hellwig:
"Fix a xen-swiotlb regression on arm64"
* tag 'dma-mapping-5.0-2' of git://git.infradead.org/users/hch/dma-mapping:
arm64/xen: fix xen-swiotlb cache flushing
+176
-102
+94
arch/arm/include/asm/xen/page-coherent.h
+94
arch/arm/include/asm/xen/page-coherent.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
3
+
#define _ASM_ARM_XEN_PAGE_COHERENT_H
4
+
5
+
#include <linux/dma-mapping.h>
6
+
#include <asm/page.h>
1
7
#include <xen/arm/page-coherent.h>
8
+
9
+
static inline const struct dma_map_ops *xen_get_dma_ops(struct device *dev)
10
+
{
11
+
if (dev && dev->archdata.dev_dma_ops)
12
+
return dev->archdata.dev_dma_ops;
13
+
return get_arch_dma_ops(NULL);
14
+
}
15
+
16
+
static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
17
+
dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
18
+
{
19
+
return xen_get_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
20
+
}
21
+
22
+
static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
23
+
void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
24
+
{
25
+
xen_get_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
26
+
}
27
+
28
+
static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
29
+
dma_addr_t dev_addr, unsigned long offset, size_t size,
30
+
enum dma_data_direction dir, unsigned long attrs)
31
+
{
32
+
unsigned long page_pfn = page_to_xen_pfn(page);
33
+
unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
34
+
unsigned long compound_pages =
35
+
(1<<compound_order(page)) * XEN_PFN_PER_PAGE;
36
+
bool local = (page_pfn <= dev_pfn) &&
37
+
(dev_pfn - page_pfn < compound_pages);
38
+
39
+
/*
40
+
* Dom0 is mapped 1:1, while the Linux page can span across
41
+
* multiple Xen pages, it's not possible for it to contain a
42
+
* mix of local and foreign Xen pages. So if the first xen_pfn
43
+
* == mfn the page is local otherwise it's a foreign page
44
+
* grant-mapped in dom0. If the page is local we can safely
45
+
* call the native dma_ops function, otherwise we call the xen
46
+
* specific function.
47
+
*/
48
+
if (local)
49
+
xen_get_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
50
+
else
51
+
__xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
52
+
}
53
+
54
+
static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
55
+
size_t size, enum dma_data_direction dir, unsigned long attrs)
56
+
{
57
+
unsigned long pfn = PFN_DOWN(handle);
58
+
/*
59
+
* Dom0 is mapped 1:1, while the Linux page can be spanned accross
60
+
* multiple Xen page, it's not possible to have a mix of local and
61
+
* foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
62
+
* foreign mfn will always return false. If the page is local we can
63
+
* safely call the native dma_ops function, otherwise we call the xen
64
+
* specific function.
65
+
*/
66
+
if (pfn_valid(pfn)) {
67
+
if (xen_get_dma_ops(hwdev)->unmap_page)
68
+
xen_get_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
69
+
} else
70
+
__xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
71
+
}
72
+
73
+
static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
74
+
dma_addr_t handle, size_t size, enum dma_data_direction dir)
75
+
{
76
+
unsigned long pfn = PFN_DOWN(handle);
77
+
if (pfn_valid(pfn)) {
78
+
if (xen_get_dma_ops(hwdev)->sync_single_for_cpu)
79
+
xen_get_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
80
+
} else
81
+
__xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
82
+
}
83
+
84
+
static inline void xen_dma_sync_single_for_device(struct device *hwdev,
85
+
dma_addr_t handle, size_t size, enum dma_data_direction dir)
86
+
{
87
+
unsigned long pfn = PFN_DOWN(handle);
88
+
if (pfn_valid(pfn)) {
89
+
if (xen_get_dma_ops(hwdev)->sync_single_for_device)
90
+
xen_get_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
91
+
} else
92
+
__xen_dma_sync_single_for_device(hwdev, handle, size, dir);
93
+
}
94
+
95
+
#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
-3
arch/arm64/include/asm/device.h
-3
arch/arm64/include/asm/device.h
+76
arch/arm64/include/asm/xen/page-coherent.h
+76
arch/arm64/include/asm/xen/page-coherent.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef _ASM_ARM64_XEN_PAGE_COHERENT_H
3
+
#define _ASM_ARM64_XEN_PAGE_COHERENT_H
4
+
5
+
#include <linux/dma-mapping.h>
6
+
#include <asm/page.h>
1
7
#include <xen/arm/page-coherent.h>
8
+
9
+
static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
10
+
dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
11
+
{
12
+
return dma_direct_alloc(hwdev, size, dma_handle, flags, attrs);
13
+
}
14
+
15
+
static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
16
+
void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
17
+
{
18
+
dma_direct_free(hwdev, size, cpu_addr, dma_handle, attrs);
19
+
}
20
+
21
+
static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
22
+
dma_addr_t handle, size_t size, enum dma_data_direction dir)
23
+
{
24
+
unsigned long pfn = PFN_DOWN(handle);
25
+
26
+
if (pfn_valid(pfn))
27
+
dma_direct_sync_single_for_cpu(hwdev, handle, size, dir);
28
+
else
29
+
__xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
30
+
}
31
+
32
+
static inline void xen_dma_sync_single_for_device(struct device *hwdev,
33
+
dma_addr_t handle, size_t size, enum dma_data_direction dir)
34
+
{
35
+
unsigned long pfn = PFN_DOWN(handle);
36
+
if (pfn_valid(pfn))
37
+
dma_direct_sync_single_for_device(hwdev, handle, size, dir);
38
+
else
39
+
__xen_dma_sync_single_for_device(hwdev, handle, size, dir);
40
+
}
41
+
42
+
static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
43
+
dma_addr_t dev_addr, unsigned long offset, size_t size,
44
+
enum dma_data_direction dir, unsigned long attrs)
45
+
{
46
+
unsigned long page_pfn = page_to_xen_pfn(page);
47
+
unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
48
+
unsigned long compound_pages =
49
+
(1<<compound_order(page)) * XEN_PFN_PER_PAGE;
50
+
bool local = (page_pfn <= dev_pfn) &&
51
+
(dev_pfn - page_pfn < compound_pages);
52
+
53
+
if (local)
54
+
dma_direct_map_page(hwdev, page, offset, size, dir, attrs);
55
+
else
56
+
__xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
57
+
}
58
+
59
+
static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
60
+
size_t size, enum dma_data_direction dir, unsigned long attrs)
61
+
{
62
+
unsigned long pfn = PFN_DOWN(handle);
63
+
/*
64
+
* Dom0 is mapped 1:1, while the Linux page can be spanned accross
65
+
* multiple Xen page, it's not possible to have a mix of local and
66
+
* foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
67
+
* foreign mfn will always return false. If the page is local we can
68
+
* safely call the native dma_ops function, otherwise we call the xen
69
+
* specific function.
70
+
*/
71
+
if (pfn_valid(pfn))
72
+
dma_direct_unmap_page(hwdev, handle, size, dir, attrs);
73
+
else
74
+
__xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
75
+
}
76
+
77
+
#endif /* _ASM_ARM64_XEN_PAGE_COHERENT_H */
+1
-3
arch/arm64/mm/dma-mapping.c
+1
-3
arch/arm64/mm/dma-mapping.c
+2
-2
drivers/xen/swiotlb-xen.c
+2
-2
drivers/xen/swiotlb-xen.c
···
645
645
void *cpu_addr, dma_addr_t dma_addr, size_t size,
646
646
unsigned long attrs)
647
647
{
648
-
#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
648
+
#ifdef CONFIG_ARM
649
649
if (xen_get_dma_ops(dev)->mmap)
650
650
return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
651
651
dma_addr, size, attrs);
···
662
662
void *cpu_addr, dma_addr_t handle, size_t size,
663
663
unsigned long attrs)
664
664
{
665
-
#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
665
+
#ifdef CONFIG_ARM
666
666
if (xen_get_dma_ops(dev)->get_sgtable) {
667
667
#if 0
668
668
/*
+3
-94
include/xen/arm/page-coherent.h
+3
-94
include/xen/arm/page-coherent.h
···
1
1
/* SPDX-License-Identifier: GPL-2.0 */
2
-
#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
3
-
#define _ASM_ARM_XEN_PAGE_COHERENT_H
4
-
5
-
#include <asm/page.h>
6
-
#include <asm/dma-mapping.h>
7
-
#include <linux/dma-mapping.h>
8
-
9
-
static inline const struct dma_map_ops *xen_get_dma_ops(struct device *dev)
10
-
{
11
-
if (dev && dev->archdata.dev_dma_ops)
12
-
return dev->archdata.dev_dma_ops;
13
-
return get_arch_dma_ops(NULL);
14
-
}
2
+
#ifndef _XEN_ARM_PAGE_COHERENT_H
3
+
#define _XEN_ARM_PAGE_COHERENT_H
15
4
16
5
void __xen_dma_map_page(struct device *hwdev, struct page *page,
17
6
dma_addr_t dev_addr, unsigned long offset, size_t size,
···
10
21
unsigned long attrs);
11
22
void __xen_dma_sync_single_for_cpu(struct device *hwdev,
12
23
dma_addr_t handle, size_t size, enum dma_data_direction dir);
13
-
14
24
void __xen_dma_sync_single_for_device(struct device *hwdev,
15
25
dma_addr_t handle, size_t size, enum dma_data_direction dir);
16
26
17
-
static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
18
-
dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
19
-
{
20
-
return xen_get_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
21
-
}
22
-
23
-
static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
24
-
void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
25
-
{
26
-
xen_get_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
27
-
}
28
-
29
-
static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
30
-
dma_addr_t dev_addr, unsigned long offset, size_t size,
31
-
enum dma_data_direction dir, unsigned long attrs)
32
-
{
33
-
unsigned long page_pfn = page_to_xen_pfn(page);
34
-
unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
35
-
unsigned long compound_pages =
36
-
(1<<compound_order(page)) * XEN_PFN_PER_PAGE;
37
-
bool local = (page_pfn <= dev_pfn) &&
38
-
(dev_pfn - page_pfn < compound_pages);
39
-
40
-
/*
41
-
* Dom0 is mapped 1:1, while the Linux page can span across
42
-
* multiple Xen pages, it's not possible for it to contain a
43
-
* mix of local and foreign Xen pages. So if the first xen_pfn
44
-
* == mfn the page is local otherwise it's a foreign page
45
-
* grant-mapped in dom0. If the page is local we can safely
46
-
* call the native dma_ops function, otherwise we call the xen
47
-
* specific function.
48
-
*/
49
-
if (local)
50
-
xen_get_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
51
-
else
52
-
__xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
53
-
}
54
-
55
-
static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
56
-
size_t size, enum dma_data_direction dir, unsigned long attrs)
57
-
{
58
-
unsigned long pfn = PFN_DOWN(handle);
59
-
/*
60
-
* Dom0 is mapped 1:1, while the Linux page can be spanned accross
61
-
* multiple Xen page, it's not possible to have a mix of local and
62
-
* foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
63
-
* foreign mfn will always return false. If the page is local we can
64
-
* safely call the native dma_ops function, otherwise we call the xen
65
-
* specific function.
66
-
*/
67
-
if (pfn_valid(pfn)) {
68
-
if (xen_get_dma_ops(hwdev)->unmap_page)
69
-
xen_get_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
70
-
} else
71
-
__xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
72
-
}
73
-
74
-
static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
75
-
dma_addr_t handle, size_t size, enum dma_data_direction dir)
76
-
{
77
-
unsigned long pfn = PFN_DOWN(handle);
78
-
if (pfn_valid(pfn)) {
79
-
if (xen_get_dma_ops(hwdev)->sync_single_for_cpu)
80
-
xen_get_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
81
-
} else
82
-
__xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
83
-
}
84
-
85
-
static inline void xen_dma_sync_single_for_device(struct device *hwdev,
86
-
dma_addr_t handle, size_t size, enum dma_data_direction dir)
87
-
{
88
-
unsigned long pfn = PFN_DOWN(handle);
89
-
if (pfn_valid(pfn)) {
90
-
if (xen_get_dma_ops(hwdev)->sync_single_for_device)
91
-
xen_get_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
92
-
} else
93
-
__xen_dma_sync_single_for_device(hwdev, handle, size, dir);
94
-
}
95
-
96
-
#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
27
+
#endif /* _XEN_ARM_PAGE_COHERENT_H */