Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommupt: Use the incoherent start/stop functions for PT_FEAT_DMA_INCOHERENT
This is the first step to supporting an incoherent walker, start and stop
the incoherence around the allocation and frees of the page table memory.
The iommu_pages API maps this to dma_map/unmap_single(), or arch cache
flushing calls.
Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
authored by
Jason Gunthorpe
and committed by
Joerg Roedel
aefd967d
bcc64b57
+88
-20
+70
-19
drivers/iommu/generic_pt/iommu_pt.h
+70
-19
drivers/iommu/generic_pt/iommu_pt.h
···
24
24
{
25
25
struct pt_common *common = common_from_iommu(iommu_table);
26
26
27
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
28
+
iommu_pages_stop_incoherent_list(free_list,
29
+
iommu_table->iommu_device);
30
+
27
31
if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) &&
28
32
iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) {
29
33
iommu_iotlb_sync(&iommu_table->domain, iotlb_gather);
···
333
329
return 0;
334
330
}
335
331
336
-
static inline struct pt_table_p *table_alloc_top(struct pt_common *common,
337
-
uintptr_t top_of_table,
338
-
gfp_t gfp)
332
+
enum alloc_mode {ALLOC_NORMAL, ALLOC_DEFER_COHERENT_FLUSH};
333
+
334
+
/* Allocate a table, the empty table will be ready to be installed. */
335
+
static inline struct pt_table_p *_table_alloc(struct pt_common *common,
336
+
size_t lg2sz, gfp_t gfp,
337
+
enum alloc_mode mode)
339
338
{
340
339
struct pt_iommu *iommu_table = iommu_from_common(common);
340
+
struct pt_table_p *table_mem;
341
341
342
+
table_mem = iommu_alloc_pages_node_sz(iommu_table->nid, gfp,
343
+
log2_to_int(lg2sz));
344
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT) &&
345
+
mode == ALLOC_NORMAL) {
346
+
int ret = iommu_pages_start_incoherent(
347
+
table_mem, iommu_table->iommu_device);
348
+
if (ret) {
349
+
iommu_free_pages(table_mem);
350
+
return ERR_PTR(ret);
351
+
}
352
+
}
353
+
return table_mem;
354
+
}
355
+
356
+
static inline struct pt_table_p *table_alloc_top(struct pt_common *common,
357
+
uintptr_t top_of_table,
358
+
gfp_t gfp,
359
+
enum alloc_mode mode)
360
+
{
342
361
/*
343
362
* Top doesn't need the free list or otherwise, so it technically
344
363
* doesn't need to use iommu pages. Use the API anyhow as the top is
345
364
* usually not smaller than PAGE_SIZE to keep things simple.
346
365
*/
347
-
return iommu_alloc_pages_node_sz(
348
-
iommu_table->nid, gfp,
349
-
log2_to_int(pt_top_memsize_lg2(common, top_of_table)));
366
+
return _table_alloc(common, pt_top_memsize_lg2(common, top_of_table),
367
+
gfp, mode);
350
368
}
351
369
352
370
/* Allocate an interior table */
353
371
static inline struct pt_table_p *table_alloc(const struct pt_state *parent_pts,
354
-
gfp_t gfp)
372
+
gfp_t gfp, enum alloc_mode mode)
355
373
{
356
-
struct pt_iommu *iommu_table =
357
-
iommu_from_common(parent_pts->range->common);
358
374
struct pt_state child_pts =
359
375
pt_init(parent_pts->range, parent_pts->level - 1, NULL);
360
376
361
-
return iommu_alloc_pages_node_sz(
362
-
iommu_table->nid, gfp,
363
-
log2_to_int(pt_num_items_lg2(&child_pts) +
364
-
ilog2(PT_ITEM_WORD_SIZE)));
377
+
return _table_alloc(parent_pts->range->common,
378
+
pt_num_items_lg2(&child_pts) +
379
+
ilog2(PT_ITEM_WORD_SIZE),
380
+
gfp, mode);
365
381
}
366
382
367
383
static inline int pt_iommu_new_table(struct pt_state *pts,
···
394
370
if (PT_WARN_ON(!pt_can_have_table(pts)))
395
371
return -ENXIO;
396
372
397
-
table_mem = table_alloc(pts, attrs->gfp);
373
+
table_mem = table_alloc(pts, attrs->gfp, ALLOC_NORMAL);
398
374
if (IS_ERR(table_mem))
399
375
return PTR_ERR(table_mem);
400
376
401
377
phys = virt_to_phys(table_mem);
402
378
if (!pt_install_table(pts, phys, attrs)) {
403
-
iommu_free_pages(table_mem);
379
+
iommu_pages_free_incoherent(
380
+
table_mem,
381
+
iommu_from_common(pts->range->common)->iommu_device);
404
382
return -EAGAIN;
405
383
}
406
384
···
415
389
pt_load_single_entry(pts);
416
390
if (PT_WARN_ON(pt_table_pa(pts) != phys)) {
417
391
pt_clear_entries(pts, ilog2(1));
418
-
iommu_free_pages(table_mem);
392
+
iommu_pages_free_incoherent(
393
+
table_mem, iommu_from_common(pts->range->common)
394
+
->iommu_device);
419
395
return -EINVAL;
420
396
}
421
397
}
···
643
615
}
644
616
645
617
new_level = pts.level;
646
-
table_mem = table_alloc_top(
647
-
common, _pt_top_set(NULL, pts.level), map->attrs.gfp);
618
+
table_mem =
619
+
table_alloc_top(common, _pt_top_set(NULL, pts.level),
620
+
map->attrs.gfp, ALLOC_DEFER_COHERENT_FLUSH);
648
621
if (IS_ERR(table_mem))
649
622
return PTR_ERR(table_mem);
650
623
iommu_pages_list_add(&free_list, table_mem);
···
660
631
pt_install_table(&pts, virt_to_phys(pts.table_lower),
661
632
&map->attrs);
662
633
new_top_of_table = _pt_top_set(pts.table, pts.level);
634
+
}
635
+
636
+
/*
637
+
* Avoid double flushing, flush it once after all pt_install_table()
638
+
*/
639
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT)) {
640
+
ret = iommu_pages_start_incoherent_list(
641
+
&free_list, iommu_table->iommu_device);
642
+
if (ret)
643
+
goto err_free;
663
644
}
664
645
665
646
/*
···
704
665
return 0;
705
666
706
667
err_free:
668
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
669
+
iommu_pages_stop_incoherent_list(&free_list,
670
+
iommu_table->iommu_device);
707
671
iommu_put_pages_list(&free_list);
708
672
return ret;
709
673
}
···
1030
988
* The driver has to already have fenced the HW access to the page table
1031
989
* and invalidated any caching referring to this memory.
1032
990
*/
991
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
992
+
iommu_pages_stop_incoherent_list(&collect.free_list,
993
+
iommu_table->iommu_device);
1033
994
iommu_put_pages_list(&collect.free_list);
1034
995
}
1035
996
···
1123
1078
memset_after(fmt_table, 0, iommu.domain);
1124
1079
1125
1080
/* The caller can initialize some of these values */
1081
+
iommu_table->iommu_device = cfg.iommu_device;
1126
1082
iommu_table->driver_ops = cfg.driver_ops;
1127
1083
iommu_table->nid = cfg.nid;
1128
1084
}
···
1169
1123
pt_feature(common, PT_FEAT_DYNAMIC_TOP)))
1170
1124
return -EINVAL;
1171
1125
1126
+
if (pt_feature(common, PT_FEAT_DMA_INCOHERENT) &&
1127
+
WARN_ON(!iommu_table->iommu_device))
1128
+
return -EINVAL;
1129
+
1172
1130
ret = pt_iommu_init_domain(iommu_table, &iommu_table->domain);
1173
1131
if (ret)
1174
1132
return ret;
1175
1133
1176
-
table_mem = table_alloc_top(common, common->top_of_table, gfp);
1134
+
table_mem = table_alloc_top(common, common->top_of_table, gfp,
1135
+
ALLOC_NORMAL);
1177
1136
if (IS_ERR(table_mem))
1178
1137
return PTR_ERR(table_mem);
1179
1138
pt_top_set(common, table_mem, pt_top_get_level(common));
+1
drivers/iommu/generic_pt/kunit_iommu.h
+1
drivers/iommu/generic_pt/kunit_iommu.h
···
139
139
140
140
priv->fmt_table.iommu.nid = NUMA_NO_NODE;
141
141
priv->fmt_table.iommu.driver_ops = &pt_kunit_driver_ops;
142
+
priv->fmt_table.iommu.iommu_device = priv->dummy_dev;
142
143
priv->domain.ops = &kunit_pt_ops;
143
144
ret = pt_iommu_init(&priv->fmt_table, &priv->cfg, GFP_KERNEL);
144
145
if (ret) {
+4
-1
drivers/iommu/generic_pt/pt_defs.h
+4
-1
drivers/iommu/generic_pt/pt_defs.h
···
48
48
/*
49
49
* When in debug mode we compile all formats with all features. This allows the
50
50
* kunit to test the full matrix. SIGN_EXTEND can't co-exist with DYNAMIC_TOP or
51
-
* FULL_VA.
51
+
* FULL_VA. DMA_INCOHERENT requires a SW bit that not all formats have
52
52
*/
53
53
#if IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)
54
54
enum {
55
55
PT_ORIG_SUPPORTED_FEATURES = PT_SUPPORTED_FEATURES,
56
56
PT_DEBUG_SUPPORTED_FEATURES =
57
57
UINT_MAX &
58
+
~((PT_ORIG_SUPPORTED_FEATURES & BIT(PT_FEAT_DMA_INCOHERENT) ?
59
+
0 :
60
+
BIT(PT_FEAT_DMA_INCOHERENT))) &
58
61
~((PT_ORIG_SUPPORTED_FEATURES & BIT(PT_FEAT_SIGN_EXTEND)) ?
59
62
BIT(PT_FEAT_DYNAMIC_TOP) | BIT(PT_FEAT_FULL_VA) :
60
63
BIT(PT_FEAT_SIGN_EXTEND)),
+6
include/linux/generic_pt/common.h
+6
include/linux/generic_pt/common.h
···
86
86
*/
87
87
enum pt_features {
88
88
/**
89
+
* @PT_FEAT_DMA_INCOHERENT: Cache flush page table memory before
90
+
* assuming the HW can read it. Otherwise a SMP release is sufficient
91
+
* for HW to read it.
92
+
*/
93
+
PT_FEAT_DMA_INCOHERENT,
94
+
/**
89
95
* @PT_FEAT_FULL_VA: The table can span the full VA range from 0 to
90
96
* PT_VADDR_MAX.
91
97
*/
+7
include/linux/generic_pt/iommu.h
+7
include/linux/generic_pt/iommu.h
···
57
57
* table walkers.
58
58
*/
59
59
int nid;
60
+
61
+
/**
62
+
* @iommu_device: Device pointer used for any DMA cache flushing when
63
+
* PT_FEAT_DMA_INCOHERENT. This is the iommu device that created the
64
+
* page table which must have dma ops that perform cache flushing.
65
+
*/
66
+
struct device *iommu_device;
60
67
};
61
68
62
69
/**