iommupt: Flush the CPU cache after any writes to the page table

Flush the CPU cache for the page table memory after each set of writes to
the page table. The iommu should have visibility to the updated entries as
soon as the map/unmap/etc operations return, like normal coherent hardware
does.

The caches also have to be flushed before any gather can be submitted to
the driver.

Implement the same solution to the race as io-pgtable-arm by using a
software PTE bit to track if a table entry has been flushed or not. If
another thread is still flushing then another concurrent map operation
could return without IOMMU visibility to a required table entry. The SW
bit will tell the second thread to also flush the cache.

Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

authored by

Jason Gunthorpe and committed by

Joerg Roedel 7 months ago efa03dab aefd967d

+61 -4

1 changed file

expand all

drivers

iommu

generic_pt

iommu_pt.h

+61 -4

drivers/iommu/generic_pt/iommu_pt.h

··· 17 17 #include <linux/cleanup.h> 18 18 #include <linux/dma-mapping.h> 19 19 20 + enum { 21 + SW_BIT_CACHE_FLUSH_DONE = 0, 22 + }; 23 + 24 + static void flush_writes_range(const struct pt_state *pts, 25 + unsigned int start_index, unsigned int end_index) 26 + { 27 + if (pts_feature(pts, PT_FEAT_DMA_INCOHERENT)) 28 + iommu_pages_flush_incoherent( 29 + iommu_from_common(pts->range->common)->iommu_device, 30 + pts->table, start_index * PT_ITEM_WORD_SIZE, 31 + (end_index - start_index) * PT_ITEM_WORD_SIZE); 32 + } 33 + 34 + static void flush_writes_item(const struct pt_state *pts) 35 + { 36 + if (pts_feature(pts, PT_FEAT_DMA_INCOHERENT)) 37 + iommu_pages_flush_incoherent( 38 + iommu_from_common(pts->range->common)->iommu_device, 39 + pts->table, pts->index * PT_ITEM_WORD_SIZE, 40 + PT_ITEM_WORD_SIZE); 41 + } 42 + 20 43 static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, 21 44 struct pt_iommu *iommu_table, pt_vaddr_t iova, 22 45 pt_vaddr_t len, ··· 218 195 dirty_len); 219 196 220 197 if (!(dirty->flags & IOMMU_DIRTY_NO_CLEAR)) { 198 + /* 199 + * No write log required because DMA incoherence and atomic 200 + * dirty tracking bits can't work together 201 + */ 221 202 pt_entry_make_write_clean(pts); 222 203 iommu_iotlb_gather_add_range(dirty->dirty->gather, 223 204 pts->range->va, dirty_len); ··· 433 406 return -EAGAIN; 434 407 } 435 408 409 + if (pts_feature(pts, PT_FEAT_DMA_INCOHERENT)) { 410 + flush_writes_item(pts); 411 + pt_set_sw_bit_release(pts, SW_BIT_CACHE_FLUSH_DONE); 412 + } 413 + 436 414 if (IS_ENABLED(CONFIG_DEBUG_GENERIC_PT)) { 437 415 /* 438 416 * The underlying table can't store the physical table address. ··· 498 466 * the gather 499 467 */ 500 468 pt_clear_entries(&pts, ilog2(1)); 469 + flush_writes_item(&pts); 501 470 502 471 iommu_pages_list_add(&collect.free_list, 503 472 pt_table_ptr(&pts)); ··· 556 523 pts.index += step; 557 524 } while (pts.index < pts.end_index); 558 525 526 + flush_writes_range(&pts, start_index, pts.index); 527 + 559 528 map->oa = oa; 560 529 return ret; 561 530 } ··· 592 557 } 593 558 } else { 594 559 pts.table_lower = pt_table_ptr(&pts); 560 + /* 561 + * Racing with a shared pt_iommu_new_table()? The other 562 + * thread is still flushing the cache, so we have to 563 + * also flush it to ensure that when our thread's map 564 + * completes all the table items leading to our mapping 565 + * are visible. 566 + * 567 + * This requires the pt_set_bit_release() to be a 568 + * release of the cache flush so that this can acquire 569 + * visibility at the iommu. 570 + */ 571 + if (pts_feature(&pts, PT_FEAT_DMA_INCOHERENT) && 572 + !pt_test_sw_bit_acquire(&pts, 573 + SW_BIT_CACHE_FLUSH_DONE)) 574 + flush_writes_item(&pts); 595 575 } 596 576 597 577 /* ··· 647 597 return -EADDRINUSE; 648 598 pt_install_leaf_entry(&pts, map->oa, PAGE_SHIFT, 649 599 &map->attrs); 600 + /* No flush, not used when incoherent */ 650 601 map->oa += PAGE_SIZE; 651 602 return 0; 652 603 } ··· 787 736 return 0; 788 737 } 789 738 790 - static int do_map(struct pt_range *range, bool single_page, 791 - struct pt_iommu_map_args *map) 739 + static int do_map(struct pt_range *range, struct pt_common *common, 740 + bool single_page, struct pt_iommu_map_args *map) 792 741 { 793 - if (single_page) { 742 + /* 743 + * The __map_single_page() fast path does not support DMA_INCOHERENT 744 + * flushing to keep its .text small. 745 + */ 746 + if (single_page && !pt_feature(common, PT_FEAT_DMA_INCOHERENT)) { 794 747 int ret; 795 748 796 749 ret = pt_walk_range(range, __map_single_page, map); ··· 897 842 898 843 PT_WARN_ON(map.leaf_level > range.top_level); 899 844 900 - ret = do_map(&range, single_page, &map); 845 + ret = do_map(&range, common, single_page, &map); 901 846 902 847 /* 903 848 * Table levels were freed and replaced with large items, flush any walk ··· 999 944 } while (true); 1000 945 1001 946 unmap->unmapped += log2_mul(num_oas, pt_table_item_lg2sz(&pts)); 947 + flush_writes_range(&pts, start_index, pts.index); 948 + 1002 949 return ret; 1003 950 } 1004 951

Configure Feed

Configure Feed