Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommupt: Directly call iommupt's unmap_range()
The common algorithm in iommupt does not require the iommu_pgsize()
calculations, it can directly unmap any arbitrary range. Add a new function
pointer to directly call an iommupt unmap_range op and make
__iommu_unmap() call it directly.
Gives about a 5% gain on single page unmappings.
The function pointer is run through pt_iommu_ops instead of
iommu_domain_ops to discourage using it outside iommupt. All drivers with
their own page tables should continue to use the simplified
map/unmap_pages() style interfaces.
Reviewed-by: Samiullah Khawaja <skhawaja@google.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
authored by
Jason Gunthorpe
and committed by
Joerg Roedel
99fb8afa
fa8fb60d
+57
-37
+4
-25
drivers/iommu/generic_pt/iommu_pt.h
+4
-25
drivers/iommu/generic_pt/iommu_pt.h
···
1031
1031
return ret;
1032
1032
}
1033
1033
1034
-
/**
1035
-
* unmap_pages() - Make a range of IOVA empty/not present
1036
-
* @domain: Domain to manipulate
1037
-
* @iova: IO virtual address to start
1038
-
* @pgsize: Length of each page
1039
-
* @pgcount: Length of the range in pgsize units starting from @iova
1040
-
* @iotlb_gather: Gather struct that must be flushed on return
1041
-
*
1042
-
* unmap_pages() will remove a translation created by map_pages(). It cannot
1043
-
* subdivide a mapping created by map_pages(), so it should be called with IOVA
1044
-
* ranges that match those passed to map_pages(). The IOVA range can aggregate
1045
-
* contiguous map_pages() calls so long as no individual range is split.
1046
-
*
1047
-
* Context: The caller must hold a write range lock that includes
1048
-
* the whole range.
1049
-
*
1050
-
* Returns: Number of bytes of VA unmapped. iova + res will be the point
1051
-
* unmapping stopped.
1052
-
*/
1053
-
size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova,
1054
-
size_t pgsize, size_t pgcount,
1034
+
static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
1035
+
dma_addr_t len,
1055
1036
struct iommu_iotlb_gather *iotlb_gather)
1056
1037
{
1057
-
struct pt_iommu *iommu_table =
1058
-
container_of(domain, struct pt_iommu, domain);
1059
1038
struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT(
1060
1039
unmap.free_list) };
1061
-
pt_vaddr_t len = pgsize * pgcount;
1062
1040
struct pt_range range;
1063
1041
int ret;
1064
1042
···
1051
1073
1052
1074
return unmap.unmapped;
1053
1075
}
1054
-
EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(unmap_pages), "GENERIC_PT_IOMMU");
1055
1076
1056
1077
static void NS(get_info)(struct pt_iommu *iommu_table,
1057
1078
struct pt_iommu_info *info)
···
1098
1121
}
1099
1122
1100
1123
static const struct pt_iommu_ops NS(ops) = {
1124
+
.unmap_range = NS(unmap_range),
1101
1125
#if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \
1102
1126
IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty)
1103
1127
.set_dirty = NS(set_dirty),
···
1161
1183
1162
1184
domain->type = __IOMMU_DOMAIN_PAGING;
1163
1185
domain->pgsize_bitmap = info.pgsize_bitmap;
1186
+
domain->is_iommupt = true;
1164
1187
1165
1188
if (pt_feature(common, PT_FEAT_DYNAMIC_TOP))
1166
1189
range = _pt_top_range(common,
+21
-6
drivers/iommu/iommu.c
+21
-6
drivers/iommu/iommu.c
···
34
34
#include <linux/sched/mm.h>
35
35
#include <linux/msi.h>
36
36
#include <uapi/linux/iommufd.h>
37
+
#include <linux/generic_pt/iommu.h>
37
38
38
39
#include "dma-iommu.h"
39
40
#include "iommu-priv.h"
···
2667
2666
}
2668
2667
EXPORT_SYMBOL_GPL(iommu_map);
2669
2668
2670
-
static size_t __iommu_unmap(struct iommu_domain *domain,
2671
-
unsigned long iova, size_t size,
2672
-
struct iommu_iotlb_gather *iotlb_gather)
2669
+
static size_t
2670
+
__iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova,
2671
+
size_t size, struct iommu_iotlb_gather *iotlb_gather)
2673
2672
{
2674
2673
const struct iommu_domain_ops *ops = domain->ops;
2675
2674
size_t unmapped_page, unmapped = 0;
2676
-
unsigned long orig_iova = iova;
2677
2675
unsigned int min_pagesz;
2678
2676
2679
2677
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
···
2718
2718
unmapped += unmapped_page;
2719
2719
}
2720
2720
2721
-
trace_unmap(orig_iova, size, unmapped);
2722
-
iommu_debug_unmap_end(domain, orig_iova, size, unmapped);
2721
+
return unmapped;
2722
+
}
2723
+
2724
+
static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova,
2725
+
size_t size,
2726
+
struct iommu_iotlb_gather *iotlb_gather)
2727
+
{
2728
+
struct pt_iommu *pt = iommupt_from_domain(domain);
2729
+
size_t unmapped;
2730
+
2731
+
if (pt)
2732
+
unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather);
2733
+
else
2734
+
unmapped = __iommu_unmap_domain_pgtbl(domain, iova, size,
2735
+
iotlb_gather);
2736
+
trace_unmap(iova, size, unmapped);
2737
+
iommu_debug_unmap_end(domain, iova, size, unmapped);
2723
2738
return unmapped;
2724
2739
}
2725
2740
+31
-6
include/linux/generic_pt/iommu.h
+31
-6
include/linux/generic_pt/iommu.h
···
66
66
struct device *iommu_device;
67
67
};
68
68
69
+
static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain)
70
+
{
71
+
if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt)
72
+
return NULL;
73
+
return container_of(domain, struct pt_iommu, domain);
74
+
}
75
+
69
76
/**
70
77
* struct pt_iommu_info - Details about the IOMMU page table
71
78
*
···
87
80
};
88
81
89
82
struct pt_iommu_ops {
83
+
/**
84
+
* @unmap_range: Make a range of IOVA empty/not present
85
+
* @iommu_table: Table to manipulate
86
+
* @iova: IO virtual address to start
87
+
* @len: Length of the range starting from @iova
88
+
* @iotlb_gather: Gather struct that must be flushed on return
89
+
*
90
+
* unmap_range() will remove a translation created by map_range(). It
91
+
* cannot subdivide a mapping created by map_range(), so it should be
92
+
* called with IOVA ranges that match those passed to map_pages. The
93
+
* IOVA range can aggregate contiguous map_range() calls so long as no
94
+
* individual range is split.
95
+
*
96
+
* Context: The caller must hold a write range lock that includes
97
+
* the whole range.
98
+
*
99
+
* Returns: Number of bytes of VA unmapped. iova + res will be the
100
+
* point unmapping stopped.
101
+
*/
102
+
size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
103
+
dma_addr_t len,
104
+
struct iommu_iotlb_gather *iotlb_gather);
105
+
90
106
/**
91
107
* @set_dirty: Make the iova write dirty
92
108
* @iommu_table: Table to manipulate
···
228
198
unsigned long iova, phys_addr_t paddr, \
229
199
size_t pgsize, size_t pgcount, \
230
200
int prot, gfp_t gfp, size_t *mapped); \
231
-
size_t pt_iommu_##fmt##_unmap_pages( \
232
-
struct iommu_domain *domain, unsigned long iova, \
233
-
size_t pgsize, size_t pgcount, \
234
-
struct iommu_iotlb_gather *iotlb_gather); \
235
201
int pt_iommu_##fmt##_read_and_clear_dirty( \
236
202
struct iommu_domain *domain, unsigned long iova, size_t size, \
237
203
unsigned long flags, struct iommu_dirty_bitmap *dirty); \
···
249
223
*/
250
224
#define IOMMU_PT_DOMAIN_OPS(fmt) \
251
225
.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \
252
-
.map_pages = &pt_iommu_##fmt##_map_pages, \
253
-
.unmap_pages = &pt_iommu_##fmt##_unmap_pages
226
+
.map_pages = &pt_iommu_##fmt##_map_pages
254
227
#define IOMMU_PT_DIRTY_OPS(fmt) \
255
228
.read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty
256
229
+1
include/linux/iommu.h
+1
include/linux/iommu.h
···
223
223
struct iommu_domain {
224
224
unsigned type;
225
225
enum iommu_domain_cookie_type cookie_type;
226
+
bool is_iommupt;
226
227
const struct iommu_domain_ops *ops;
227
228
const struct iommu_dirty_ops *dirty_ops;
228
229
const struct iommu_ops *owner; /* Whose domain_alloc we came from */