Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommupt: Add iova_to_phys op
iova_to_phys is a performance path for the DMA API and iommufd, implement
it using an unrolled get_user_pages() like function waterfall scheme.
The implementation itself is fairly trivial.
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Pasha Tatashin <pasha.tatashin@soleen.com>
Reviewed-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Tested-by: Pasha Tatashin <pasha.tatashin@soleen.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
authored by
Jason Gunthorpe
and committed by
Joerg Roedel
9d4c274c
879ced2b
+119
-5
+105
drivers/iommu/generic_pt/iommu_pt.h
+105
drivers/iommu/generic_pt/iommu_pt.h
···
17
17
18
18
#define DOMAIN_NS(op) CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), op)
19
19
20
+
static int make_range_ul(struct pt_common *common, struct pt_range *range,
21
+
unsigned long iova, unsigned long len)
22
+
{
23
+
unsigned long last;
24
+
25
+
if (unlikely(len == 0))
26
+
return -EINVAL;
27
+
28
+
if (check_add_overflow(iova, len - 1, &last))
29
+
return -EOVERFLOW;
30
+
31
+
*range = pt_make_range(common, iova, last);
32
+
if (sizeof(iova) > sizeof(range->va)) {
33
+
if (unlikely(range->va != iova || range->last_va != last))
34
+
return -EOVERFLOW;
35
+
}
36
+
return 0;
37
+
}
38
+
39
+
static __maybe_unused int make_range_u64(struct pt_common *common,
40
+
struct pt_range *range, u64 iova,
41
+
u64 len)
42
+
{
43
+
if (unlikely(iova > ULONG_MAX || len > ULONG_MAX))
44
+
return -EOVERFLOW;
45
+
return make_range_ul(common, range, iova, len);
46
+
}
47
+
48
+
/*
49
+
* Some APIs use unsigned long, while othersuse dma_addr_t as the type. Dispatch
50
+
* to the correct validation based on the type.
51
+
*/
52
+
#define make_range_no_check(common, range, iova, len) \
53
+
({ \
54
+
int ret; \
55
+
if (sizeof(iova) > sizeof(unsigned long) || \
56
+
sizeof(len) > sizeof(unsigned long)) \
57
+
ret = make_range_u64(common, range, iova, len); \
58
+
else \
59
+
ret = make_range_ul(common, range, iova, len); \
60
+
ret; \
61
+
})
62
+
63
+
#define make_range(common, range, iova, len) \
64
+
({ \
65
+
int ret = make_range_no_check(common, range, iova, len); \
66
+
if (!ret) \
67
+
ret = pt_check_range(range); \
68
+
ret; \
69
+
})
70
+
71
+
static __always_inline int __do_iova_to_phys(struct pt_range *range, void *arg,
72
+
unsigned int level,
73
+
struct pt_table_p *table,
74
+
pt_level_fn_t descend_fn)
75
+
{
76
+
struct pt_state pts = pt_init(range, level, table);
77
+
pt_oaddr_t *res = arg;
78
+
79
+
switch (pt_load_single_entry(&pts)) {
80
+
case PT_ENTRY_EMPTY:
81
+
return -ENOENT;
82
+
case PT_ENTRY_TABLE:
83
+
return pt_descend(&pts, arg, descend_fn);
84
+
case PT_ENTRY_OA:
85
+
*res = pt_entry_oa_exact(&pts);
86
+
return 0;
87
+
}
88
+
return -ENOENT;
89
+
}
90
+
PT_MAKE_LEVELS(__iova_to_phys, __do_iova_to_phys);
91
+
92
+
/**
93
+
* iova_to_phys() - Return the output address for the given IOVA
94
+
* @iommu_table: Table to query
95
+
* @iova: IO virtual address to query
96
+
*
97
+
* Determine the output address from the given IOVA. @iova may have any
98
+
* alignment, the returned physical will be adjusted with any sub page offset.
99
+
*
100
+
* Context: The caller must hold a read range lock that includes @iova.
101
+
*
102
+
* Return: 0 if there is no translation for the given iova.
103
+
*/
104
+
phys_addr_t DOMAIN_NS(iova_to_phys)(struct iommu_domain *domain,
105
+
dma_addr_t iova)
106
+
{
107
+
struct pt_iommu *iommu_table =
108
+
container_of(domain, struct pt_iommu, domain);
109
+
struct pt_range range;
110
+
pt_oaddr_t res;
111
+
int ret;
112
+
113
+
ret = make_range(common_from_iommu(iommu_table), &range, iova, 1);
114
+
if (ret)
115
+
return ret;
116
+
117
+
ret = pt_walk_range(&range, __iova_to_phys, &res);
118
+
/* PHYS_ADDR_MAX would be a better error code */
119
+
if (ret)
120
+
return 0;
121
+
return res;
122
+
}
123
+
EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(iova_to_phys), "GENERIC_PT_IOMMU");
124
+
20
125
struct pt_iommu_collect_args {
21
126
struct iommu_pages_list free_list;
22
127
};
+14
-5
include/linux/generic_pt/iommu.h
+14
-5
include/linux/generic_pt/iommu.h
···
116
116
};
117
117
118
118
/* Generate the exported function signatures from iommu_pt.h */
119
-
#define IOMMU_PROTOTYPES(fmt) \
120
-
int pt_iommu_##fmt##_init(struct pt_iommu_##fmt *table, \
121
-
const struct pt_iommu_##fmt##_cfg *cfg, \
122
-
gfp_t gfp); \
123
-
void pt_iommu_##fmt##_hw_info(struct pt_iommu_##fmt *table, \
119
+
#define IOMMU_PROTOTYPES(fmt) \
120
+
phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \
121
+
dma_addr_t iova); \
122
+
int pt_iommu_##fmt##_init(struct pt_iommu_##fmt *table, \
123
+
const struct pt_iommu_##fmt##_cfg *cfg, \
124
+
gfp_t gfp); \
125
+
void pt_iommu_##fmt##_hw_info(struct pt_iommu_##fmt *table, \
124
126
struct pt_iommu_##fmt##_hw_info *info)
125
127
#define IOMMU_FORMAT(fmt, member) \
126
128
struct pt_iommu_##fmt { \
···
130
128
struct pt_##fmt member; \
131
129
}; \
132
130
IOMMU_PROTOTYPES(fmt)
131
+
132
+
/*
133
+
* A driver uses IOMMU_PT_DOMAIN_OPS to populate the iommu_domain_ops for the
134
+
* iommu_pt
135
+
*/
136
+
#define IOMMU_PT_DOMAIN_OPS(fmt) \
137
+
.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys,
133
138
134
139
/*
135
140
* The driver should setup its domain struct like