Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommupt: Add map_pages op
map is slightly complicated because it has to handle a number of special
edge cases:
- Overmapping a previously shared, but now empty, table level with an OA.
Requries validating and freeing the possibly empty tables
- Doing the above across an entire to-be-created contiguous entry
- Installing a new shared table level concurrently with another thread
- Expanding the table by adding more top levels
Table expansion is a unique feature of AMDv1, this version is quite
similar except we handle racing concurrent lockless map. The table top
pointer and starting level are encoded in a single uintptr_t which ensures
we can READ_ONCE() without tearing. Any op will do the READ_ONCE() and use
that fixed point as its starting point. Concurrent expansion is handled
with a table global spinlock.
When inserting a new table entry map checks that the entire portion of the
table is empty. This includes freeing any empty lower tables that will be
overwritten by an OA. A separate free list is used while checking and
collecting all the empty lower tables so that writing the new entry is
uninterrupted, either the new entry fully writes or nothing changes.
A special fast path for PAGE_SIZE is implemented that does a direct walk
to the leaf level and installs a single entry. This gives ~15% improvement
for iommu_map() when mapping lists of single pages.
This version sits under the iommu_domain_ops as map_pages() but does not
require the external page size calculation. The implementation is actually
map_range() and can do arbitrary ranges, internally handling all the
validation and supporting any arrangment of page sizes. A future series
can optimize iommu_map() to take advantage of this.
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Tested-by: Pasha Tatashin <pasha.tatashin@soleen.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
authored by
Jason Gunthorpe
and committed by
Joerg Roedel
dcd6a011
7c53f423
+560
-2
+500
-1
drivers/iommu/generic_pt/iommu_pt.h
+500
-1
drivers/iommu/generic_pt/iommu_pt.h
···
91
91
ret; \
92
92
})
93
93
94
+
static inline unsigned int compute_best_pgsize(struct pt_state *pts,
95
+
pt_oaddr_t oa)
96
+
{
97
+
struct pt_iommu *iommu_table = iommu_from_common(pts->range->common);
98
+
99
+
if (!pt_can_have_leaf(pts))
100
+
return 0;
101
+
102
+
/*
103
+
* The page size is limited by the domain's bitmap. This allows the core
104
+
* code to reduce the supported page sizes by changing the bitmap.
105
+
*/
106
+
return pt_compute_best_pgsize(pt_possible_sizes(pts) &
107
+
iommu_table->domain.pgsize_bitmap,
108
+
pts->range->va, pts->range->last_va, oa);
109
+
}
110
+
94
111
static __always_inline int __do_iova_to_phys(struct pt_range *range, void *arg,
95
112
unsigned int level,
96
113
struct pt_table_p *table,
···
164
147
165
148
struct pt_iommu_collect_args {
166
149
struct iommu_pages_list free_list;
150
+
/* Fail if any OAs are within the range */
151
+
u8 check_mapped : 1;
167
152
};
168
153
169
154
static int __collect_tables(struct pt_range *range, void *arg,
···
175
156
struct pt_iommu_collect_args *collect = arg;
176
157
int ret;
177
158
178
-
if (!pt_can_have_table(&pts))
159
+
if (!collect->check_mapped && !pt_can_have_table(&pts))
179
160
return 0;
180
161
181
162
for_each_pt_level_entry(&pts) {
···
186
167
return ret;
187
168
continue;
188
169
}
170
+
if (pts.type == PT_ENTRY_OA && collect->check_mapped)
171
+
return -EADDRINUSE;
189
172
}
190
173
return 0;
191
174
}
···
207
186
iommu_table->nid, gfp,
208
187
log2_to_int(pt_top_memsize_lg2(common, top_of_table)));
209
188
}
189
+
190
+
/* Allocate an interior table */
191
+
static inline struct pt_table_p *table_alloc(const struct pt_state *parent_pts,
192
+
gfp_t gfp)
193
+
{
194
+
struct pt_iommu *iommu_table =
195
+
iommu_from_common(parent_pts->range->common);
196
+
struct pt_state child_pts =
197
+
pt_init(parent_pts->range, parent_pts->level - 1, NULL);
198
+
199
+
return iommu_alloc_pages_node_sz(
200
+
iommu_table->nid, gfp,
201
+
log2_to_int(pt_num_items_lg2(&child_pts) +
202
+
ilog2(PT_ITEM_WORD_SIZE)));
203
+
}
204
+
205
+
static inline int pt_iommu_new_table(struct pt_state *pts,
206
+
struct pt_write_attrs *attrs)
207
+
{
208
+
struct pt_table_p *table_mem;
209
+
phys_addr_t phys;
210
+
211
+
/* Given PA/VA/length can't be represented */
212
+
if (PT_WARN_ON(!pt_can_have_table(pts)))
213
+
return -ENXIO;
214
+
215
+
table_mem = table_alloc(pts, attrs->gfp);
216
+
if (IS_ERR(table_mem))
217
+
return PTR_ERR(table_mem);
218
+
219
+
phys = virt_to_phys(table_mem);
220
+
if (!pt_install_table(pts, phys, attrs)) {
221
+
iommu_free_pages(table_mem);
222
+
return -EAGAIN;
223
+
}
224
+
225
+
if (IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)) {
226
+
/*
227
+
* The underlying table can't store the physical table address.
228
+
* This happens when kunit testing tables outside their normal
229
+
* environment where a CPU might be limited.
230
+
*/
231
+
pt_load_single_entry(pts);
232
+
if (PT_WARN_ON(pt_table_pa(pts) != phys)) {
233
+
pt_clear_entries(pts, ilog2(1));
234
+
iommu_free_pages(table_mem);
235
+
return -EINVAL;
236
+
}
237
+
}
238
+
239
+
pts->table_lower = table_mem;
240
+
return 0;
241
+
}
242
+
243
+
struct pt_iommu_map_args {
244
+
struct iommu_iotlb_gather *iotlb_gather;
245
+
struct pt_write_attrs attrs;
246
+
pt_oaddr_t oa;
247
+
unsigned int leaf_pgsize_lg2;
248
+
unsigned int leaf_level;
249
+
};
250
+
251
+
/*
252
+
* This will recursively check any tables in the block to validate they are
253
+
* empty and then free them through the gather.
254
+
*/
255
+
static int clear_contig(const struct pt_state *start_pts,
256
+
struct iommu_iotlb_gather *iotlb_gather,
257
+
unsigned int step, unsigned int pgsize_lg2)
258
+
{
259
+
struct pt_iommu *iommu_table =
260
+
iommu_from_common(start_pts->range->common);
261
+
struct pt_range range = *start_pts->range;
262
+
struct pt_state pts =
263
+
pt_init(&range, start_pts->level, start_pts->table);
264
+
struct pt_iommu_collect_args collect = { .check_mapped = true };
265
+
int ret;
266
+
267
+
pts.index = start_pts->index;
268
+
pts.end_index = start_pts->index + step;
269
+
for (; _pt_iter_load(&pts); pt_next_entry(&pts)) {
270
+
if (pts.type == PT_ENTRY_TABLE) {
271
+
collect.free_list =
272
+
IOMMU_PAGES_LIST_INIT(collect.free_list);
273
+
ret = pt_walk_descend_all(&pts, __collect_tables,
274
+
&collect);
275
+
if (ret)
276
+
return ret;
277
+
278
+
/*
279
+
* The table item must be cleared before we can update
280
+
* the gather
281
+
*/
282
+
pt_clear_entries(&pts, ilog2(1));
283
+
284
+
iommu_pages_list_add(&collect.free_list,
285
+
pt_table_ptr(&pts));
286
+
gather_range_pages(
287
+
iotlb_gather, iommu_table, range.va,
288
+
log2_to_int(pt_table_item_lg2sz(&pts)),
289
+
&collect.free_list);
290
+
} else if (pts.type != PT_ENTRY_EMPTY) {
291
+
return -EADDRINUSE;
292
+
}
293
+
}
294
+
return 0;
295
+
}
296
+
297
+
static int __map_range_leaf(struct pt_range *range, void *arg,
298
+
unsigned int level, struct pt_table_p *table)
299
+
{
300
+
struct pt_state pts = pt_init(range, level, table);
301
+
struct pt_iommu_map_args *map = arg;
302
+
unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2;
303
+
unsigned int start_index;
304
+
pt_oaddr_t oa = map->oa;
305
+
unsigned int step;
306
+
bool need_contig;
307
+
int ret = 0;
308
+
309
+
PT_WARN_ON(map->leaf_level != level);
310
+
PT_WARN_ON(!pt_can_have_leaf(&pts));
311
+
312
+
step = log2_to_int_t(unsigned int,
313
+
leaf_pgsize_lg2 - pt_table_item_lg2sz(&pts));
314
+
need_contig = leaf_pgsize_lg2 != pt_table_item_lg2sz(&pts);
315
+
316
+
_pt_iter_first(&pts);
317
+
start_index = pts.index;
318
+
do {
319
+
pts.type = pt_load_entry_raw(&pts);
320
+
if (pts.type != PT_ENTRY_EMPTY || need_contig) {
321
+
if (pts.index != start_index)
322
+
pt_index_to_va(&pts);
323
+
ret = clear_contig(&pts, map->iotlb_gather, step,
324
+
leaf_pgsize_lg2);
325
+
if (ret)
326
+
break;
327
+
}
328
+
329
+
if (IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)) {
330
+
pt_index_to_va(&pts);
331
+
PT_WARN_ON(compute_best_pgsize(&pts, oa) !=
332
+
leaf_pgsize_lg2);
333
+
}
334
+
pt_install_leaf_entry(&pts, oa, leaf_pgsize_lg2, &map->attrs);
335
+
336
+
oa += log2_to_int(leaf_pgsize_lg2);
337
+
pts.index += step;
338
+
} while (pts.index < pts.end_index);
339
+
340
+
map->oa = oa;
341
+
return ret;
342
+
}
343
+
344
+
static int __map_range(struct pt_range *range, void *arg, unsigned int level,
345
+
struct pt_table_p *table)
346
+
{
347
+
struct pt_state pts = pt_init(range, level, table);
348
+
struct pt_iommu_map_args *map = arg;
349
+
int ret;
350
+
351
+
PT_WARN_ON(map->leaf_level == level);
352
+
PT_WARN_ON(!pt_can_have_table(&pts));
353
+
354
+
_pt_iter_first(&pts);
355
+
356
+
/* Descend to a child table */
357
+
do {
358
+
pts.type = pt_load_entry_raw(&pts);
359
+
360
+
if (pts.type != PT_ENTRY_TABLE) {
361
+
if (pts.type != PT_ENTRY_EMPTY)
362
+
return -EADDRINUSE;
363
+
ret = pt_iommu_new_table(&pts, &map->attrs);
364
+
if (ret) {
365
+
/*
366
+
* Racing with another thread installing a table
367
+
*/
368
+
if (ret == -EAGAIN)
369
+
continue;
370
+
return ret;
371
+
}
372
+
} else {
373
+
pts.table_lower = pt_table_ptr(&pts);
374
+
}
375
+
376
+
/*
377
+
* The already present table can possibly be shared with another
378
+
* concurrent map.
379
+
*/
380
+
if (map->leaf_level == level - 1)
381
+
ret = pt_descend(&pts, arg, __map_range_leaf);
382
+
else
383
+
ret = pt_descend(&pts, arg, __map_range);
384
+
if (ret)
385
+
return ret;
386
+
387
+
pts.index++;
388
+
pt_index_to_va(&pts);
389
+
if (pts.index >= pts.end_index)
390
+
break;
391
+
} while (true);
392
+
return 0;
393
+
}
394
+
395
+
/*
396
+
* Fast path for the easy case of mapping a 4k page to an already allocated
397
+
* table. This is a common workload. If it returns EAGAIN run the full algorithm
398
+
* instead.
399
+
*/
400
+
static __always_inline int __do_map_single_page(struct pt_range *range,
401
+
void *arg, unsigned int level,
402
+
struct pt_table_p *table,
403
+
pt_level_fn_t descend_fn)
404
+
{
405
+
struct pt_state pts = pt_init(range, level, table);
406
+
struct pt_iommu_map_args *map = arg;
407
+
408
+
pts.type = pt_load_single_entry(&pts);
409
+
if (level == 0) {
410
+
if (pts.type != PT_ENTRY_EMPTY)
411
+
return -EADDRINUSE;
412
+
pt_install_leaf_entry(&pts, map->oa, PAGE_SHIFT,
413
+
&map->attrs);
414
+
map->oa += PAGE_SIZE;
415
+
return 0;
416
+
}
417
+
if (pts.type == PT_ENTRY_TABLE)
418
+
return pt_descend(&pts, arg, descend_fn);
419
+
/* Something else, use the slow path */
420
+
return -EAGAIN;
421
+
}
422
+
PT_MAKE_LEVELS(__map_single_page, __do_map_single_page);
423
+
424
+
/*
425
+
* Add a table to the top, increasing the top level as much as necessary to
426
+
* encompass range.
427
+
*/
428
+
static int increase_top(struct pt_iommu *iommu_table, struct pt_range *range,
429
+
struct pt_iommu_map_args *map)
430
+
{
431
+
struct iommu_pages_list free_list = IOMMU_PAGES_LIST_INIT(free_list);
432
+
struct pt_common *common = common_from_iommu(iommu_table);
433
+
uintptr_t top_of_table = READ_ONCE(common->top_of_table);
434
+
uintptr_t new_top_of_table = top_of_table;
435
+
struct pt_table_p *table_mem;
436
+
unsigned int new_level;
437
+
spinlock_t *domain_lock;
438
+
unsigned long flags;
439
+
int ret;
440
+
441
+
while (true) {
442
+
struct pt_range top_range =
443
+
_pt_top_range(common, new_top_of_table);
444
+
struct pt_state pts = pt_init_top(&top_range);
445
+
446
+
top_range.va = range->va;
447
+
top_range.last_va = range->last_va;
448
+
449
+
if (!pt_check_range(&top_range) && map->leaf_level <= pts.level)
450
+
break;
451
+
452
+
pts.level++;
453
+
if (pts.level > PT_MAX_TOP_LEVEL ||
454
+
pt_table_item_lg2sz(&pts) >= common->max_vasz_lg2) {
455
+
ret = -ERANGE;
456
+
goto err_free;
457
+
}
458
+
459
+
new_level = pts.level;
460
+
table_mem = table_alloc_top(
461
+
common, _pt_top_set(NULL, pts.level), map->attrs.gfp);
462
+
if (IS_ERR(table_mem))
463
+
return PTR_ERR(table_mem);
464
+
iommu_pages_list_add(&free_list, table_mem);
465
+
466
+
/* The new table links to the lower table always at index 0 */
467
+
top_range.va = 0;
468
+
top_range.top_level = new_level;
469
+
pts.table_lower = pts.table;
470
+
pts.table = table_mem;
471
+
pt_load_single_entry(&pts);
472
+
PT_WARN_ON(pts.index != 0);
473
+
pt_install_table(&pts, virt_to_phys(pts.table_lower),
474
+
&map->attrs);
475
+
new_top_of_table = _pt_top_set(pts.table, pts.level);
476
+
}
477
+
478
+
/*
479
+
* top_of_table is write locked by the spinlock, but readers can use
480
+
* READ_ONCE() to get the value. Since we encode both the level and the
481
+
* pointer in one quanta the lockless reader will always see something
482
+
* valid. The HW must be updated to the new level under the spinlock
483
+
* before top_of_table is updated so that concurrent readers don't map
484
+
* into the new level until it is fully functional. If another thread
485
+
* already updated it while we were working then throw everything away
486
+
* and try again.
487
+
*/
488
+
domain_lock = iommu_table->driver_ops->get_top_lock(iommu_table);
489
+
spin_lock_irqsave(domain_lock, flags);
490
+
if (common->top_of_table != top_of_table) {
491
+
spin_unlock_irqrestore(domain_lock, flags);
492
+
ret = -EAGAIN;
493
+
goto err_free;
494
+
}
495
+
496
+
/*
497
+
* We do not issue any flushes for change_top on the expectation that
498
+
* any walk cache will not become a problem by adding another layer to
499
+
* the tree. Misses will rewalk from the updated top pointer, hits
500
+
* continue to be correct. Negative caching is fine too since all the
501
+
* new IOVA added by the new top is non-present.
502
+
*/
503
+
iommu_table->driver_ops->change_top(
504
+
iommu_table, virt_to_phys(table_mem), new_level);
505
+
WRITE_ONCE(common->top_of_table, new_top_of_table);
506
+
spin_unlock_irqrestore(domain_lock, flags);
507
+
return 0;
508
+
509
+
err_free:
510
+
iommu_put_pages_list(&free_list);
511
+
return ret;
512
+
}
513
+
514
+
static int check_map_range(struct pt_iommu *iommu_table, struct pt_range *range,
515
+
struct pt_iommu_map_args *map)
516
+
{
517
+
struct pt_common *common = common_from_iommu(iommu_table);
518
+
int ret;
519
+
520
+
do {
521
+
ret = pt_check_range(range);
522
+
if (!pt_feature(common, PT_FEAT_DYNAMIC_TOP))
523
+
return ret;
524
+
525
+
if (!ret && map->leaf_level <= range->top_level)
526
+
break;
527
+
528
+
ret = increase_top(iommu_table, range, map);
529
+
if (ret && ret != -EAGAIN)
530
+
return ret;
531
+
532
+
/* Reload the new top */
533
+
*range = pt_make_range(common, range->va, range->last_va);
534
+
} while (ret);
535
+
PT_WARN_ON(pt_check_range(range));
536
+
return 0;
537
+
}
538
+
539
+
static int do_map(struct pt_range *range, bool single_page,
540
+
struct pt_iommu_map_args *map)
541
+
{
542
+
if (single_page) {
543
+
int ret;
544
+
545
+
ret = pt_walk_range(range, __map_single_page, map);
546
+
if (ret != -EAGAIN)
547
+
return ret;
548
+
/* EAGAIN falls through to the full path */
549
+
}
550
+
551
+
if (map->leaf_level == range->top_level)
552
+
return pt_walk_range(range, __map_range_leaf, map);
553
+
return pt_walk_range(range, __map_range, map);
554
+
}
555
+
556
+
/**
557
+
* map_pages() - Install translation for an IOVA range
558
+
* @domain: Domain to manipulate
559
+
* @iova: IO virtual address to start
560
+
* @paddr: Physical/Output address to start
561
+
* @pgsize: Length of each page
562
+
* @pgcount: Length of the range in pgsize units starting from @iova
563
+
* @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO
564
+
* @gfp: GFP flags for any memory allocations
565
+
* @mapped: Total bytes successfully mapped
566
+
*
567
+
* The range starting at IOVA will have paddr installed into it. The caller
568
+
* must specify a valid pgsize and pgcount to segment the range into compatible
569
+
* blocks.
570
+
*
571
+
* On error the caller will probably want to invoke unmap on the range from iova
572
+
* up to the amount indicated by @mapped to return the table back to an
573
+
* unchanged state.
574
+
*
575
+
* Context: The caller must hold a write range lock that includes the whole
576
+
* range.
577
+
*
578
+
* Returns: -ERRNO on failure, 0 on success. The number of bytes of VA that were
579
+
* mapped are added to @mapped, @mapped is not zerod first.
580
+
*/
581
+
int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova,
582
+
phys_addr_t paddr, size_t pgsize, size_t pgcount,
583
+
int prot, gfp_t gfp, size_t *mapped)
584
+
{
585
+
struct pt_iommu *iommu_table =
586
+
container_of(domain, struct pt_iommu, domain);
587
+
pt_vaddr_t pgsize_bitmap = iommu_table->domain.pgsize_bitmap;
588
+
struct pt_common *common = common_from_iommu(iommu_table);
589
+
struct iommu_iotlb_gather iotlb_gather;
590
+
pt_vaddr_t len = pgsize * pgcount;
591
+
struct pt_iommu_map_args map = {
592
+
.iotlb_gather = &iotlb_gather,
593
+
.oa = paddr,
594
+
.leaf_pgsize_lg2 = vaffs(pgsize),
595
+
};
596
+
bool single_page = false;
597
+
struct pt_range range;
598
+
int ret;
599
+
600
+
iommu_iotlb_gather_init(&iotlb_gather);
601
+
602
+
if (WARN_ON(!(prot & (IOMMU_READ | IOMMU_WRITE))))
603
+
return -EINVAL;
604
+
605
+
/* Check the paddr doesn't exceed what the table can store */
606
+
if ((sizeof(pt_oaddr_t) < sizeof(paddr) &&
607
+
(pt_vaddr_t)paddr > PT_VADDR_MAX) ||
608
+
(common->max_oasz_lg2 != PT_VADDR_MAX_LG2 &&
609
+
oalog2_div(paddr, common->max_oasz_lg2)))
610
+
return -ERANGE;
611
+
612
+
ret = pt_iommu_set_prot(common, &map.attrs, prot);
613
+
if (ret)
614
+
return ret;
615
+
map.attrs.gfp = gfp;
616
+
617
+
ret = make_range_no_check(common, &range, iova, len);
618
+
if (ret)
619
+
return ret;
620
+
621
+
/* Calculate target page size and level for the leaves */
622
+
if (pt_has_system_page_size(common) && pgsize == PAGE_SIZE &&
623
+
pgcount == 1) {
624
+
PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE));
625
+
if (log2_mod(iova | paddr, PAGE_SHIFT))
626
+
return -ENXIO;
627
+
map.leaf_pgsize_lg2 = PAGE_SHIFT;
628
+
map.leaf_level = 0;
629
+
single_page = true;
630
+
} else {
631
+
map.leaf_pgsize_lg2 = pt_compute_best_pgsize(
632
+
pgsize_bitmap, range.va, range.last_va, paddr);
633
+
if (!map.leaf_pgsize_lg2)
634
+
return -ENXIO;
635
+
map.leaf_level =
636
+
pt_pgsz_lg2_to_level(common, map.leaf_pgsize_lg2);
637
+
}
638
+
639
+
ret = check_map_range(iommu_table, &range, &map);
640
+
if (ret)
641
+
return ret;
642
+
643
+
PT_WARN_ON(map.leaf_level > range.top_level);
644
+
645
+
ret = do_map(&range, single_page, &map);
646
+
647
+
/*
648
+
* Table levels were freed and replaced with large items, flush any walk
649
+
* cache that may refer to the freed levels.
650
+
*/
651
+
if (!iommu_pages_list_empty(&iotlb_gather.freelist))
652
+
iommu_iotlb_sync(&iommu_table->domain, &iotlb_gather);
653
+
654
+
/* Bytes successfully mapped */
655
+
PT_WARN_ON(!ret && map.oa - paddr != len);
656
+
*mapped += map.oa - paddr;
657
+
return ret;
658
+
}
659
+
EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(map_pages), "GENERIC_PT_IOMMU");
210
660
211
661
struct pt_unmap_args {
212
662
struct iommu_pages_list free_list;
···
937
445
memset_after(fmt_table, 0, iommu.domain);
938
446
939
447
/* The caller can initialize some of these values */
448
+
iommu_table->driver_ops = cfg.driver_ops;
940
449
iommu_table->nid = cfg.nid;
941
450
}
942
451
···
970
477
ret = pt_init_common(common);
971
478
if (ret)
972
479
return ret;
480
+
481
+
if (pt_feature(common, PT_FEAT_DYNAMIC_TOP) &&
482
+
WARN_ON(!iommu_table->driver_ops ||
483
+
!iommu_table->driver_ops->change_top ||
484
+
!iommu_table->driver_ops->get_top_lock))
485
+
return -EINVAL;
973
486
974
487
if (pt_feature(common, PT_FEAT_SIGN_EXTEND) &&
975
488
(pt_feature(common, PT_FEAT_FULL_VA) ||
+1
-1
drivers/iommu/generic_pt/pt_iter.h
+1
-1
drivers/iommu/generic_pt/pt_iter.h
···
612
612
* This builds a function call tree that can be fully inlined.
613
613
* The caller must provide a function body in an __always_inline function::
614
614
*
615
-
* static __always_inline int do(struct pt_range *range, void *arg,
615
+
* static __always_inline int do_fn(struct pt_range *range, void *arg,
616
616
* unsigned int level, struct pt_table_p *table,
617
617
* pt_level_fn_t descend_fn)
618
618
*
+59
include/linux/generic_pt/iommu.h
+59
include/linux/generic_pt/iommu.h
···
11
11
12
12
struct iommu_iotlb_gather;
13
13
struct pt_iommu_ops;
14
+
struct pt_iommu_driver_ops;
14
15
15
16
/**
16
17
* DOC: IOMMU Radix Page Table
···
43
42
* @ops: Function pointers to access the API
44
43
*/
45
44
const struct pt_iommu_ops *ops;
45
+
46
+
/**
47
+
* @driver_ops: Function pointers provided by the HW driver to help
48
+
* manage HW details like caches.
49
+
*/
50
+
const struct pt_iommu_driver_ops *driver_ops;
46
51
47
52
/**
48
53
* @nid: Node ID to use for table memory allocations. The IOMMU driver
···
91
84
void (*deinit)(struct pt_iommu *iommu_table);
92
85
};
93
86
87
+
/**
88
+
* struct pt_iommu_driver_ops - HW IOTLB cache flushing operations
89
+
*
90
+
* The IOMMU driver should implement these using container_of(iommu_table) to
91
+
* get to it's iommu_domain derived structure. All ops can be called in atomic
92
+
* contexts as they are buried under DMA API calls.
93
+
*/
94
+
struct pt_iommu_driver_ops {
95
+
/**
96
+
* @change_top: Update the top of table pointer
97
+
* @iommu_table: Table to operate on
98
+
* @top_paddr: New CPU physical address of the top pointer
99
+
* @top_level: IOMMU PT level of the new top
100
+
*
101
+
* Called under the get_top_lock() spinlock. The driver must update all
102
+
* HW references to this domain with a new top address and
103
+
* configuration. On return mappings placed in the new top must be
104
+
* reachable by the HW.
105
+
*
106
+
* top_level encodes the level in IOMMU PT format, level 0 is the
107
+
* smallest page size increasing from there. This has to be translated
108
+
* to any HW specific format. During this call the new top will not be
109
+
* visible to any other API.
110
+
*
111
+
* This op is only used by PT_FEAT_DYNAMIC_TOP, and is required if
112
+
* enabled.
113
+
*/
114
+
void (*change_top)(struct pt_iommu *iommu_table, phys_addr_t top_paddr,
115
+
unsigned int top_level);
116
+
117
+
/**
118
+
* @get_top_lock: lock to hold when changing the table top
119
+
* @iommu_table: Table to operate on
120
+
*
121
+
* Return a lock to hold when changing the table top page table from
122
+
* being stored in HW. The lock will be held prior to calling
123
+
* change_top() and released once the top is fully visible.
124
+
*
125
+
* Typically this would be a lock that protects the iommu_domain's
126
+
* attachment list.
127
+
*
128
+
* This op is only used by PT_FEAT_DYNAMIC_TOP, and is required if
129
+
* enabled.
130
+
*/
131
+
spinlock_t *(*get_top_lock)(struct pt_iommu *iommu_table);
132
+
};
133
+
94
134
static inline void pt_iommu_deinit(struct pt_iommu *iommu_table)
95
135
{
96
136
/*
···
174
120
#define IOMMU_PROTOTYPES(fmt) \
175
121
phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \
176
122
dma_addr_t iova); \
123
+
int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain, \
124
+
unsigned long iova, phys_addr_t paddr, \
125
+
size_t pgsize, size_t pgcount, \
126
+
int prot, gfp_t gfp, size_t *mapped); \
177
127
size_t pt_iommu_##fmt##_unmap_pages( \
178
128
struct iommu_domain *domain, unsigned long iova, \
179
129
size_t pgsize, size_t pgcount, \
···
200
142
*/
201
143
#define IOMMU_PT_DOMAIN_OPS(fmt) \
202
144
.iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \
145
+
.map_pages = &pt_iommu_##fmt##_map_pages, \
203
146
.unmap_pages = &pt_iommu_##fmt##_unmap_pages
204
147
205
148
/*