Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-6.9-2024-03-24' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping fixes from Christoph Hellwig:
"This has a set of swiotlb alignment fixes for sometimes very long
standing bugs from Will. We've been discussion them for a while and
they should be solid now"
* tag 'dma-mapping-6.9-2024-03-24' of git://git.infradead.org/users/hch/dma-mapping:
swiotlb: Reinstate page-alignment for mappings >= PAGE_SIZE
iommu/dma: Force swiotlb_max_mapping_size on an untrusted device
swiotlb: Fix alignment checks when both allocation and DMA masks are present
swiotlb: Honour dma_alloc_coherent() alignment in swiotlb_alloc()
swiotlb: Enforce page alignment in swiotlb_alloc()
swiotlb: Fix double-allocation of slots due to broken alignment handling
+43
-13
+9
drivers/iommu/dma-iommu.c
+9
drivers/iommu/dma-iommu.c
···
1711
1711
return iova_rcache_range();
1712
1712
}
1713
1713
1714
+
static size_t iommu_dma_max_mapping_size(struct device *dev)
1715
+
{
1716
+
if (dev_is_untrusted(dev))
1717
+
return swiotlb_max_mapping_size(dev);
1718
+
1719
+
return SIZE_MAX;
1720
+
}
1721
+
1714
1722
static const struct dma_map_ops iommu_dma_ops = {
1715
1723
.flags = DMA_F_PCI_P2PDMA_SUPPORTED,
1716
1724
.alloc = iommu_dma_alloc,
···
1741
1733
.unmap_resource = iommu_dma_unmap_resource,
1742
1734
.get_merge_boundary = iommu_dma_get_merge_boundary,
1743
1735
.opt_mapping_size = iommu_dma_opt_mapping_size,
1736
+
.max_mapping_size = iommu_dma_max_mapping_size,
1744
1737
};
1745
1738
1746
1739
/*
+34
-13
kernel/dma/swiotlb.c
+34
-13
kernel/dma/swiotlb.c
···
1003
1003
dma_addr_t tbl_dma_addr =
1004
1004
phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
1005
1005
unsigned long max_slots = get_max_slots(boundary_mask);
1006
-
unsigned int iotlb_align_mask =
1007
-
dma_get_min_align_mask(dev) | alloc_align_mask;
1006
+
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1008
1007
unsigned int nslots = nr_slots(alloc_size), stride;
1009
1008
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
1010
1009
unsigned int index, slots_checked, count = 0, i;
···
1015
1016
BUG_ON(area_index >= pool->nareas);
1016
1017
1017
1018
/*
1018
-
* For allocations of PAGE_SIZE or larger only look for page aligned
1019
-
* allocations.
1019
+
* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1020
+
* page-aligned in the absence of any other alignment requirements.
1021
+
* 'alloc_align_mask' was later introduced to specify the alignment
1022
+
* explicitly, however this is passed as zero for streaming mappings
1023
+
* and so we preserve the old behaviour there in case any drivers are
1024
+
* relying on it.
1020
1025
*/
1021
-
if (alloc_size >= PAGE_SIZE)
1022
-
iotlb_align_mask |= ~PAGE_MASK;
1023
-
iotlb_align_mask &= ~(IO_TLB_SIZE - 1);
1026
+
if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1027
+
alloc_align_mask = PAGE_SIZE - 1;
1028
+
1029
+
/*
1030
+
* Ensure that the allocation is at least slot-aligned and update
1031
+
* 'iotlb_align_mask' to ignore bits that will be preserved when
1032
+
* offsetting into the allocation.
1033
+
*/
1034
+
alloc_align_mask |= (IO_TLB_SIZE - 1);
1035
+
iotlb_align_mask &= ~alloc_align_mask;
1024
1036
1025
1037
/*
1026
1038
* For mappings with an alignment requirement don't bother looping to
1027
1039
* unaligned slots once we found an aligned one.
1028
1040
*/
1029
-
stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
1041
+
stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1030
1042
1031
1043
spin_lock_irqsave(&area->lock, flags);
1032
1044
if (unlikely(nslots > pool->area_nslabs - area->used))
···
1047
1037
index = area->index;
1048
1038
1049
1039
for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1050
-
slot_index = slot_base + index;
1040
+
phys_addr_t tlb_addr;
1051
1041
1052
-
if (orig_addr &&
1053
-
(slot_addr(tbl_dma_addr, slot_index) &
1054
-
iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
1042
+
slot_index = slot_base + index;
1043
+
tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1044
+
1045
+
if ((tlb_addr & alloc_align_mask) ||
1046
+
(orig_addr && (tlb_addr & iotlb_align_mask) !=
1047
+
(orig_addr & iotlb_align_mask))) {
1055
1048
index = wrap_area_index(pool, index + 1);
1056
1049
slots_checked++;
1057
1050
continue;
···
1690
1677
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1691
1678
struct io_tlb_pool *pool;
1692
1679
phys_addr_t tlb_addr;
1680
+
unsigned int align;
1693
1681
int index;
1694
1682
1695
1683
if (!mem)
1696
1684
return NULL;
1697
1685
1698
-
index = swiotlb_find_slots(dev, 0, size, 0, &pool);
1686
+
align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1687
+
index = swiotlb_find_slots(dev, 0, size, align, &pool);
1699
1688
if (index == -1)
1700
1689
return NULL;
1701
1690
1702
1691
tlb_addr = slot_addr(pool->start, index);
1692
+
if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1693
+
dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1694
+
&tlb_addr);
1695
+
swiotlb_release_slots(dev, tlb_addr);
1696
+
return NULL;
1697
+
}
1703
1698
1704
1699
return pfn_to_page(PFN_DOWN(tlb_addr));
1705
1700
}