Merge git://git.infradead.org/iommu-2.6

+2 -13

arch/x86/Kconfig

··· 1913 1913 recommended you say N here while the DMAR code remains 1914 1914 experimental. 1915 1915 1916 - config DMAR_GFX_WA 1917 - def_bool y 1918 - prompt "Support for Graphics workaround" 1919 - depends on DMAR 1920 - ---help--- 1921 - Current Graphics drivers tend to use physical address 1922 - for DMA and avoid using DMA APIs. Setting this config 1923 - option permits the IOMMU driver to set a unity map for 1924 - all the OS-visible memory. Hence the driver can continue 1925 - to use physical addresses for DMA. 1926 - 1927 1916 config DMAR_FLOPPY_WA 1928 1917 def_bool y 1929 1918 depends on DMAR 1930 1919 ---help--- 1931 - Floppy disk drivers are know to bypass DMA API calls 1920 + Floppy disk drivers are known to bypass DMA API calls 1932 1921 thereby failing to work when IOMMU is enabled. This 1933 1922 workaround will setup a 1:1 mapping for the first 1934 - 16M to make floppy (an ISA device) work. 1923 + 16MiB to make floppy (an ISA device) work. 1935 1924 1936 1925 config INTR_REMAP 1937 1926 bool "Support for Interrupt Remapping (EXPERIMENTAL)"

+1 -1

arch/x86/kernel/pci-dma.c

··· 211 211 #ifdef CONFIG_SWIOTLB 212 212 if (!strncmp(p, "soft", 4)) 213 213 swiotlb = 1; 214 + #endif 214 215 if (!strncmp(p, "pt", 2)) { 215 216 iommu_pass_through = 1; 216 217 return 1; 217 218 } 218 - #endif 219 219 220 220 gart_parse_options(p); 221 221

+351 -346

drivers/pci/intel-iommu.c

··· 56 56 #define MAX_AGAW_WIDTH 64 57 57 58 58 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1) 59 + #define DOMAIN_MAX_PFN(gaw) ((((u64)1) << (gaw-VTD_PAGE_SHIFT)) - 1) 59 60 60 61 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) 61 62 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) 62 63 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64)) 63 64 64 - #ifndef PHYSICAL_PAGE_MASK 65 - #define PHYSICAL_PAGE_MASK PAGE_MASK 66 - #endif 65 + 66 + /* VT-d pages must always be _smaller_ than MM pages. Otherwise things 67 + are never going to work. */ 68 + static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn) 69 + { 70 + return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT); 71 + } 72 + 73 + static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn) 74 + { 75 + return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT); 76 + } 77 + static inline unsigned long page_to_dma_pfn(struct page *pg) 78 + { 79 + return mm_to_dma_pfn(page_to_pfn(pg)); 80 + } 81 + static inline unsigned long virt_to_dma_pfn(void *p) 82 + { 83 + return page_to_dma_pfn(virt_to_page(p)); 84 + } 67 85 68 86 /* global iommu list, set NULL for ignored DMAR units */ 69 87 static struct intel_iommu **g_iommus; ··· 222 204 223 205 static inline u64 dma_pte_addr(struct dma_pte *pte) 224 206 { 225 - return (pte->val & VTD_PAGE_MASK); 207 + #ifdef CONFIG_64BIT 208 + return pte->val & VTD_PAGE_MASK; 209 + #else 210 + /* Must have a full atomic 64-bit read */ 211 + return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK; 212 + #endif 226 213 } 227 214 228 - static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr) 215 + static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn) 229 216 { 230 - pte->val |= (addr & VTD_PAGE_MASK); 217 + pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT; 231 218 } 232 219 233 220 static inline bool dma_pte_present(struct dma_pte *pte) 234 221 { 235 222 return (pte->val & 3) != 0; 223 + } 224 + 225 + static inline int first_pte_in_page(struct dma_pte *pte) 226 + { 227 + return !((unsigned long)pte & ~VTD_PAGE_MASK); 236 228 } 237 229 238 230 /* ··· 272 244 struct iova_domain iovad; /* iova's that belong to this domain */ 273 245 274 246 struct dma_pte *pgd; /* virtual address */ 275 - spinlock_t mapping_lock; /* page table lock */ 276 247 int gaw; /* max guest address width */ 277 248 278 249 /* adjusted guest address width, 0 is level 2 30-bit */ ··· 675 648 676 649 static inline unsigned int level_to_offset_bits(int level) 677 650 { 678 - return (12 + (level - 1) * LEVEL_STRIDE); 651 + return (level - 1) * LEVEL_STRIDE; 679 652 } 680 653 681 - static inline int address_level_offset(u64 addr, int level) 654 + static inline int pfn_level_offset(unsigned long pfn, int level) 682 655 { 683 - return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK); 656 + return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK; 684 657 } 685 658 686 - static inline u64 level_mask(int level) 659 + static inline unsigned long level_mask(int level) 687 660 { 688 - return ((u64)-1 << level_to_offset_bits(level)); 661 + return -1UL << level_to_offset_bits(level); 689 662 } 690 663 691 - static inline u64 level_size(int level) 664 + static inline unsigned long level_size(int level) 692 665 { 693 - return ((u64)1 << level_to_offset_bits(level)); 666 + return 1UL << level_to_offset_bits(level); 694 667 } 695 668 696 - static inline u64 align_to_level(u64 addr, int level) 669 + static inline unsigned long align_to_level(unsigned long pfn, int level) 697 670 { 698 - return ((addr + level_size(level) - 1) & level_mask(level)); 671 + return (pfn + level_size(level) - 1) & level_mask(level); 699 672 } 700 673 701 - static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr) 674 + static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, 675 + unsigned long pfn) 702 676 { 703 - int addr_width = agaw_to_width(domain->agaw); 677 + int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; 704 678 struct dma_pte *parent, *pte = NULL; 705 679 int level = agaw_to_level(domain->agaw); 706 680 int offset; 707 - unsigned long flags; 708 681 709 682 BUG_ON(!domain->pgd); 710 - 711 - addr &= (((u64)1) << addr_width) - 1; 683 + BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width); 712 684 parent = domain->pgd; 713 685 714 - spin_lock_irqsave(&domain->mapping_lock, flags); 715 686 while (level > 0) { 716 687 void *tmp_page; 717 688 718 - offset = address_level_offset(addr, level); 689 + offset = pfn_level_offset(pfn, level); 719 690 pte = &parent[offset]; 720 691 if (level == 1) 721 692 break; 722 693 723 694 if (!dma_pte_present(pte)) { 695 + uint64_t pteval; 696 + 724 697 tmp_page = alloc_pgtable_page(); 725 698 726 - if (!tmp_page) { 727 - spin_unlock_irqrestore(&domain->mapping_lock, 728 - flags); 699 + if (!tmp_page) 729 700 return NULL; 701 + 702 + domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE); 703 + pteval = (virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE; 704 + if (cmpxchg64(&pte->val, 0ULL, pteval)) { 705 + /* Someone else set it while we were thinking; use theirs. */ 706 + free_pgtable_page(tmp_page); 707 + } else { 708 + dma_pte_addr(pte); 709 + domain_flush_cache(domain, pte, sizeof(*pte)); 730 710 } 731 - domain_flush_cache(domain, tmp_page, PAGE_SIZE); 732 - dma_set_pte_addr(pte, virt_to_phys(tmp_page)); 733 - /* 734 - * high level table always sets r/w, last level page 735 - * table control read/write 736 - */ 737 - dma_set_pte_readable(pte); 738 - dma_set_pte_writable(pte); 739 - domain_flush_cache(domain, pte, sizeof(*pte)); 740 711 } 741 712 parent = phys_to_virt(dma_pte_addr(pte)); 742 713 level--; 743 714 } 744 715 745 - spin_unlock_irqrestore(&domain->mapping_lock, flags); 746 716 return pte; 747 717 } 748 718 749 719 /* return address's pte at specific level */ 750 - static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr, 751 - int level) 720 + static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain, 721 + unsigned long pfn, 722 + int level) 752 723 { 753 724 struct dma_pte *parent, *pte = NULL; 754 725 int total = agaw_to_level(domain->agaw); ··· 754 729 755 730 parent = domain->pgd; 756 731 while (level <= total) { 757 - offset = address_level_offset(addr, total); 732 + offset = pfn_level_offset(pfn, total); 758 733 pte = &parent[offset]; 759 734 if (level == total) 760 735 return pte; ··· 767 742 return NULL; 768 743 } 769 744 770 - /* clear one page's page table */ 771 - static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr) 772 - { 773 - struct dma_pte *pte = NULL; 774 - 775 - /* get last level pte */ 776 - pte = dma_addr_level_pte(domain, addr, 1); 777 - 778 - if (pte) { 779 - dma_clear_pte(pte); 780 - domain_flush_cache(domain, pte, sizeof(*pte)); 781 - } 782 - } 783 - 784 745 /* clear last level pte, a tlb flush should be followed */ 785 - static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end) 746 + static void dma_pte_clear_range(struct dmar_domain *domain, 747 + unsigned long start_pfn, 748 + unsigned long last_pfn) 786 749 { 787 - int addr_width = agaw_to_width(domain->agaw); 788 - int npages; 750 + int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; 751 + struct dma_pte *first_pte, *pte; 789 752 790 - start &= (((u64)1) << addr_width) - 1; 791 - end &= (((u64)1) << addr_width) - 1; 792 - /* in case it's partial page */ 793 - start &= PAGE_MASK; 794 - end = PAGE_ALIGN(end); 795 - npages = (end - start) / VTD_PAGE_SIZE; 753 + BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); 754 + BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); 796 755 797 - /* we don't need lock here, nobody else touches the iova range */ 798 - while (npages--) { 799 - dma_pte_clear_one(domain, start); 800 - start += VTD_PAGE_SIZE; 756 + /* we don't need lock here; nobody else touches the iova range */ 757 + while (start_pfn <= last_pfn) { 758 + first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1); 759 + if (!pte) { 760 + start_pfn = align_to_level(start_pfn + 1, 2); 761 + continue; 762 + } 763 + do { 764 + dma_clear_pte(pte); 765 + start_pfn++; 766 + pte++; 767 + } while (start_pfn <= last_pfn && !first_pte_in_page(pte)); 768 + 769 + domain_flush_cache(domain, first_pte, 770 + (void *)pte - (void *)first_pte); 801 771 } 802 772 } 803 773 804 774 /* free page table pages. last level pte should already be cleared */ 805 775 static void dma_pte_free_pagetable(struct dmar_domain *domain, 806 - u64 start, u64 end) 776 + unsigned long start_pfn, 777 + unsigned long last_pfn) 807 778 { 808 - int addr_width = agaw_to_width(domain->agaw); 809 - struct dma_pte *pte; 779 + int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; 780 + struct dma_pte *first_pte, *pte; 810 781 int total = agaw_to_level(domain->agaw); 811 782 int level; 812 - u64 tmp; 783 + unsigned long tmp; 813 784 814 - start &= (((u64)1) << addr_width) - 1; 815 - end &= (((u64)1) << addr_width) - 1; 785 + BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); 786 + BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); 816 787 817 - /* we don't need lock here, nobody else touches the iova range */ 788 + /* We don't need lock here; nobody else touches the iova range */ 818 789 level = 2; 819 790 while (level <= total) { 820 - tmp = align_to_level(start, level); 821 - if (tmp >= end || (tmp + level_size(level) > end)) 791 + tmp = align_to_level(start_pfn, level); 792 + 793 + /* If we can't even clear one PTE at this level, we're done */ 794 + if (tmp + level_size(level) - 1 > last_pfn) 822 795 return; 823 796 824 - while (tmp < end) { 825 - pte = dma_addr_level_pte(domain, tmp, level); 826 - if (pte) { 827 - free_pgtable_page( 828 - phys_to_virt(dma_pte_addr(pte))); 829 - dma_clear_pte(pte); 830 - domain_flush_cache(domain, pte, sizeof(*pte)); 797 + while (tmp + level_size(level) - 1 <= last_pfn) { 798 + first_pte = pte = dma_pfn_level_pte(domain, tmp, level); 799 + if (!pte) { 800 + tmp = align_to_level(tmp + 1, level + 1); 801 + continue; 831 802 } 832 - tmp += level_size(level); 803 + do { 804 + if (dma_pte_present(pte)) { 805 + free_pgtable_page(phys_to_virt(dma_pte_addr(pte))); 806 + dma_clear_pte(pte); 807 + } 808 + pte++; 809 + tmp += level_size(level); 810 + } while (!first_pte_in_page(pte) && 811 + tmp + level_size(level) - 1 <= last_pfn); 812 + 813 + domain_flush_cache(domain, first_pte, 814 + (void *)pte - (void *)first_pte); 815 + 833 816 } 834 817 level++; 835 818 } 836 819 /* free pgd */ 837 - if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) { 820 + if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { 838 821 free_pgtable_page(domain->pgd); 839 822 domain->pgd = NULL; 840 823 } ··· 1068 1035 } 1069 1036 1070 1037 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, 1071 - u64 addr, unsigned int pages) 1038 + unsigned long pfn, unsigned int pages) 1072 1039 { 1073 1040 unsigned int mask = ilog2(__roundup_pow_of_two(pages)); 1041 + uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; 1074 1042 1075 - BUG_ON(addr & (~VTD_PAGE_MASK)); 1076 1043 BUG_ON(pages == 0); 1077 1044 1078 1045 /* ··· 1087 1054 else 1088 1055 iommu->flush.flush_iotlb(iommu, did, addr, mask, 1089 1056 DMA_TLB_PSI_FLUSH); 1090 - if (did) 1057 + 1058 + /* 1059 + * In caching mode, domain ID 0 is reserved for non-present to present 1060 + * mapping flush. Device IOTLB doesn't need to be flushed in this case. 1061 + */ 1062 + if (!cap_caching_mode(iommu->cap) || did) 1091 1063 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask); 1092 1064 } 1093 1065 ··· 1317 1279 struct pci_dev *pdev = NULL; 1318 1280 struct iova *iova; 1319 1281 int i; 1320 - u64 addr, size; 1321 1282 1322 1283 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN); 1323 1284 ··· 1339 1302 r = &pdev->resource[i]; 1340 1303 if (!r->flags || !(r->flags & IORESOURCE_MEM)) 1341 1304 continue; 1342 - addr = r->start; 1343 - addr &= PHYSICAL_PAGE_MASK; 1344 - size = r->end - addr; 1345 - size = PAGE_ALIGN(size); 1346 - iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr), 1347 - IOVA_PFN(size + addr) - 1); 1305 + iova = reserve_iova(&reserved_iova_list, 1306 + IOVA_PFN(r->start), 1307 + IOVA_PFN(r->end)); 1348 1308 if (!iova) 1349 1309 printk(KERN_ERR "Reserve iova failed\n"); 1350 1310 } ··· 1375 1341 unsigned long sagaw; 1376 1342 1377 1343 init_iova_domain(&domain->iovad, DMA_32BIT_PFN); 1378 - spin_lock_init(&domain->mapping_lock); 1379 1344 spin_lock_init(&domain->iommu_lock); 1380 1345 1381 1346 domain_reserve_special_ranges(domain); ··· 1421 1388 { 1422 1389 struct dmar_drhd_unit *drhd; 1423 1390 struct intel_iommu *iommu; 1424 - u64 end; 1425 1391 1426 1392 /* Domain 0 is reserved, so dont process it */ 1427 1393 if (!domain) ··· 1429 1397 domain_remove_dev_info(domain); 1430 1398 /* destroy iovas */ 1431 1399 put_iova_domain(&domain->iovad); 1432 - end = DOMAIN_MAX_ADDR(domain->gaw); 1433 - end = end & (~PAGE_MASK); 1434 1400 1435 1401 /* clear ptes */ 1436 - dma_pte_clear_range(domain, 0, end); 1402 + dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); 1437 1403 1438 1404 /* free page tables */ 1439 - dma_pte_free_pagetable(domain, 0, end); 1405 + dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); 1440 1406 1441 1407 for_each_active_iommu(iommu, drhd) 1442 1408 if (test_bit(iommu->seq_id, &domain->iommu_bmp)) ··· 1648 1618 tmp->devfn); 1649 1619 } 1650 1620 1651 - static int 1652 - domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova, 1653 - u64 hpa, size_t size, int prot) 1621 + static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 1622 + struct scatterlist *sg, unsigned long phys_pfn, 1623 + unsigned long nr_pages, int prot) 1654 1624 { 1655 - u64 start_pfn, end_pfn; 1656 - struct dma_pte *pte; 1657 - int index; 1658 - int addr_width = agaw_to_width(domain->agaw); 1625 + struct dma_pte *first_pte = NULL, *pte = NULL; 1626 + phys_addr_t uninitialized_var(pteval); 1627 + int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; 1628 + unsigned long sg_res; 1659 1629 1660 - hpa &= (((u64)1) << addr_width) - 1; 1630 + BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width); 1661 1631 1662 1632 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0) 1663 1633 return -EINVAL; 1664 - iova &= PAGE_MASK; 1665 - start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT; 1666 - end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT; 1667 - index = 0; 1668 - while (start_pfn < end_pfn) { 1669 - pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index); 1670 - if (!pte) 1671 - return -ENOMEM; 1634 + 1635 + prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP; 1636 + 1637 + if (sg) 1638 + sg_res = 0; 1639 + else { 1640 + sg_res = nr_pages + 1; 1641 + pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot; 1642 + } 1643 + 1644 + while (nr_pages--) { 1645 + uint64_t tmp; 1646 + 1647 + if (!sg_res) { 1648 + sg_res = (sg->offset + sg->length + VTD_PAGE_SIZE - 1) >> VTD_PAGE_SHIFT; 1649 + sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset; 1650 + sg->dma_length = sg->length; 1651 + pteval = page_to_phys(sg_page(sg)) | prot; 1652 + } 1653 + if (!pte) { 1654 + first_pte = pte = pfn_to_dma_pte(domain, iov_pfn); 1655 + if (!pte) 1656 + return -ENOMEM; 1657 + } 1672 1658 /* We don't need lock here, nobody else 1673 1659 * touches the iova range 1674 1660 */ 1675 - BUG_ON(dma_pte_addr(pte)); 1676 - dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT); 1677 - dma_set_pte_prot(pte, prot); 1678 - if (prot & DMA_PTE_SNP) 1679 - dma_set_pte_snp(pte); 1680 - domain_flush_cache(domain, pte, sizeof(*pte)); 1681 - start_pfn++; 1682 - index++; 1661 + tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); 1662 + if (tmp) { 1663 + static int dumps = 5; 1664 + printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", 1665 + iov_pfn, tmp, (unsigned long long)pteval); 1666 + if (dumps) { 1667 + dumps--; 1668 + debug_dma_dump_mappings(NULL); 1669 + } 1670 + WARN_ON(1); 1671 + } 1672 + pte++; 1673 + if (!nr_pages || first_pte_in_page(pte)) { 1674 + domain_flush_cache(domain, first_pte, 1675 + (void *)pte - (void *)first_pte); 1676 + pte = NULL; 1677 + } 1678 + iov_pfn++; 1679 + pteval += VTD_PAGE_SIZE; 1680 + sg_res--; 1681 + if (!sg_res) 1682 + sg = sg_next(sg); 1683 1683 } 1684 1684 return 0; 1685 + } 1686 + 1687 + static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 1688 + struct scatterlist *sg, unsigned long nr_pages, 1689 + int prot) 1690 + { 1691 + return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot); 1692 + } 1693 + 1694 + static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn, 1695 + unsigned long phys_pfn, unsigned long nr_pages, 1696 + int prot) 1697 + { 1698 + return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot); 1685 1699 } 1686 1700 1687 1701 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn) ··· 1918 1844 1919 1845 static int iommu_identity_mapping; 1920 1846 1847 + static int iommu_domain_identity_map(struct dmar_domain *domain, 1848 + unsigned long long start, 1849 + unsigned long long end) 1850 + { 1851 + unsigned long first_vpfn = start >> VTD_PAGE_SHIFT; 1852 + unsigned long last_vpfn = end >> VTD_PAGE_SHIFT; 1853 + 1854 + if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn), 1855 + dma_to_mm_pfn(last_vpfn))) { 1856 + printk(KERN_ERR "IOMMU: reserve iova failed\n"); 1857 + return -ENOMEM; 1858 + } 1859 + 1860 + pr_debug("Mapping reserved region %llx-%llx for domain %d\n", 1861 + start, end, domain->id); 1862 + /* 1863 + * RMRR range might have overlap with physical memory range, 1864 + * clear it first 1865 + */ 1866 + dma_pte_clear_range(domain, first_vpfn, last_vpfn); 1867 + 1868 + return domain_pfn_mapping(domain, first_vpfn, first_vpfn, 1869 + last_vpfn - first_vpfn + 1, 1870 + DMA_PTE_READ|DMA_PTE_WRITE); 1871 + } 1872 + 1921 1873 static int iommu_prepare_identity_map(struct pci_dev *pdev, 1922 1874 unsigned long long start, 1923 1875 unsigned long long end) 1924 1876 { 1925 1877 struct dmar_domain *domain; 1926 - unsigned long size; 1927 - unsigned long long base; 1928 1878 int ret; 1929 1879 1930 1880 printk(KERN_INFO 1931 - "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n", 1932 - pci_name(pdev), start, end); 1933 - if (iommu_identity_mapping) 1934 - domain = si_domain; 1935 - else 1936 - /* page table init */ 1937 - domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH); 1881 + "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n", 1882 + pci_name(pdev), start, end); 1883 + 1884 + domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH); 1938 1885 if (!domain) 1939 1886 return -ENOMEM; 1940 1887 1941 - /* The address might not be aligned */ 1942 - base = start & PAGE_MASK; 1943 - size = end - base; 1944 - size = PAGE_ALIGN(size); 1945 - if (!reserve_iova(&domain->iovad, IOVA_PFN(base), 1946 - IOVA_PFN(base + size) - 1)) { 1947 - printk(KERN_ERR "IOMMU: reserve iova failed\n"); 1948 - ret = -ENOMEM; 1949 - goto error; 1950 - } 1951 - 1952 - pr_debug("Mapping reserved region %lx@%llx for %s\n", 1953 - size, base, pci_name(pdev)); 1954 - /* 1955 - * RMRR range might have overlap with physical memory range, 1956 - * clear it first 1957 - */ 1958 - dma_pte_clear_range(domain, base, base + size); 1959 - 1960 - ret = domain_page_mapping(domain, base, base, size, 1961 - DMA_PTE_READ|DMA_PTE_WRITE); 1888 + ret = iommu_domain_identity_map(domain, start, end); 1962 1889 if (ret) 1963 1890 goto error; 1964 1891 1965 1892 /* context entry init */ 1966 1893 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL); 1967 - if (!ret) 1968 - return 0; 1969 - error: 1894 + if (ret) 1895 + goto error; 1896 + 1897 + return 0; 1898 + 1899 + error: 1970 1900 domain_exit(domain); 1971 1901 return ret; 1972 - 1973 1902 } 1974 1903 1975 1904 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr, ··· 1984 1907 rmrr->end_address + 1); 1985 1908 } 1986 1909 1987 - struct iommu_prepare_data { 1988 - struct pci_dev *pdev; 1989 - int ret; 1990 - }; 1991 - 1992 - static int __init iommu_prepare_work_fn(unsigned long start_pfn, 1993 - unsigned long end_pfn, void *datax) 1994 - { 1995 - struct iommu_prepare_data *data; 1996 - 1997 - data = (struct iommu_prepare_data *)datax; 1998 - 1999 - data->ret = iommu_prepare_identity_map(data->pdev, 2000 - start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT); 2001 - return data->ret; 2002 - 2003 - } 2004 - 2005 - static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev) 2006 - { 2007 - int nid; 2008 - struct iommu_prepare_data data; 2009 - 2010 - data.pdev = pdev; 2011 - data.ret = 0; 2012 - 2013 - for_each_online_node(nid) { 2014 - work_with_active_regions(nid, iommu_prepare_work_fn, &data); 2015 - if (data.ret) 2016 - return data.ret; 2017 - } 2018 - return data.ret; 2019 - } 2020 - 2021 - #ifdef CONFIG_DMAR_GFX_WA 2022 - static void __init iommu_prepare_gfx_mapping(void) 2023 - { 2024 - struct pci_dev *pdev = NULL; 2025 - int ret; 2026 - 2027 - for_each_pci_dev(pdev) { 2028 - if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO || 2029 - !IS_GFX_DEVICE(pdev)) 2030 - continue; 2031 - printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n", 2032 - pci_name(pdev)); 2033 - ret = iommu_prepare_with_active_regions(pdev); 2034 - if (ret) 2035 - printk(KERN_ERR "IOMMU: mapping reserved region failed\n"); 2036 - } 2037 - } 2038 - #else /* !CONFIG_DMAR_GFX_WA */ 2039 - static inline void iommu_prepare_gfx_mapping(void) 2040 - { 2041 - return; 2042 - } 2043 - #endif 2044 - 2045 1910 #ifdef CONFIG_DMAR_FLOPPY_WA 2046 1911 static inline void iommu_prepare_isa(void) 2047 1912 { ··· 1994 1975 if (!pdev) 1995 1976 return; 1996 1977 1997 - printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n"); 1978 + printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n"); 1998 1979 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024); 1999 1980 2000 1981 if (ret) 2001 - printk(KERN_ERR "IOMMU: Failed to create 0-64M identity map, " 2002 - "floppy might not work\n"); 1982 + printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; " 1983 + "floppy might not work\n"); 2003 1984 2004 1985 } 2005 1986 #else ··· 2027 2008 } 2028 2009 2029 2010 static int md_domain_init(struct dmar_domain *domain, int guest_width); 2011 + 2012 + static int __init si_domain_work_fn(unsigned long start_pfn, 2013 + unsigned long end_pfn, void *datax) 2014 + { 2015 + int *ret = datax; 2016 + 2017 + *ret = iommu_domain_identity_map(si_domain, 2018 + (uint64_t)start_pfn << PAGE_SHIFT, 2019 + (uint64_t)end_pfn << PAGE_SHIFT); 2020 + return *ret; 2021 + 2022 + } 2023 + 2030 2024 static int si_domain_init(void) 2031 2025 { 2032 2026 struct dmar_drhd_unit *drhd; 2033 2027 struct intel_iommu *iommu; 2034 - int ret = 0; 2028 + int nid, ret = 0; 2035 2029 2036 2030 si_domain = alloc_domain(); 2037 2031 if (!si_domain) 2038 2032 return -EFAULT; 2039 2033 2034 + pr_debug("Identity mapping domain is domain %d\n", si_domain->id); 2040 2035 2041 2036 for_each_active_iommu(iommu, drhd) { 2042 2037 ret = iommu_attach_domain(si_domain, iommu); ··· 2066 2033 } 2067 2034 2068 2035 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY; 2036 + 2037 + for_each_online_node(nid) { 2038 + work_with_active_regions(nid, si_domain_work_fn, &ret); 2039 + if (ret) 2040 + return ret; 2041 + } 2069 2042 2070 2043 return 0; 2071 2044 } ··· 2126 2087 if (ret) 2127 2088 return -EFAULT; 2128 2089 2129 - printk(KERN_INFO "IOMMU: Setting identity map:\n"); 2130 2090 for_each_pci_dev(pdev) { 2131 - ret = iommu_prepare_with_active_regions(pdev); 2132 - if (ret) { 2133 - printk(KERN_INFO "1:1 mapping to one domain failed.\n"); 2134 - return -EFAULT; 2135 - } 2091 + printk(KERN_INFO "IOMMU: identity mapping for device %s\n", 2092 + pci_name(pdev)); 2093 + 2094 + ret = domain_context_mapping(si_domain, pdev, 2095 + CONTEXT_TT_MULTI_LEVEL); 2096 + if (ret) 2097 + return ret; 2136 2098 ret = domain_add_dev_info(si_domain, pdev); 2137 2099 if (ret) 2138 2100 return ret; ··· 2324 2284 } 2325 2285 } 2326 2286 2327 - iommu_prepare_gfx_mapping(); 2328 - 2329 2287 iommu_prepare_isa(); 2330 2288 } 2331 2289 ··· 2368 2330 return ret; 2369 2331 } 2370 2332 2371 - static inline u64 aligned_size(u64 host_addr, size_t size) 2333 + static inline unsigned long aligned_nrpages(unsigned long host_addr, 2334 + size_t size) 2372 2335 { 2373 - u64 addr; 2374 - addr = (host_addr & (~PAGE_MASK)) + size; 2375 - return PAGE_ALIGN(addr); 2336 + host_addr &= ~PAGE_MASK; 2337 + host_addr += size + PAGE_SIZE - 1; 2338 + 2339 + return host_addr >> VTD_PAGE_SHIFT; 2376 2340 } 2377 2341 2378 - struct iova * 2379 - iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end) 2380 - { 2381 - struct iova *piova; 2382 - 2383 - /* Make sure it's in range */ 2384 - end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end); 2385 - if (!size || (IOVA_START_ADDR + size > end)) 2386 - return NULL; 2387 - 2388 - piova = alloc_iova(&domain->iovad, 2389 - size >> PAGE_SHIFT, IOVA_PFN(end), 1); 2390 - return piova; 2391 - } 2392 - 2393 - static struct iova * 2394 - __intel_alloc_iova(struct device *dev, struct dmar_domain *domain, 2395 - size_t size, u64 dma_mask) 2342 + static struct iova *intel_alloc_iova(struct device *dev, 2343 + struct dmar_domain *domain, 2344 + unsigned long nrpages, uint64_t dma_mask) 2396 2345 { 2397 2346 struct pci_dev *pdev = to_pci_dev(dev); 2398 2347 struct iova *iova = NULL; 2399 2348 2400 - if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac) 2401 - iova = iommu_alloc_iova(domain, size, dma_mask); 2402 - else { 2349 + /* Restrict dma_mask to the width that the iommu can handle */ 2350 + dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask); 2351 + 2352 + if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) { 2403 2353 /* 2404 2354 * First try to allocate an io virtual address in 2405 2355 * DMA_BIT_MASK(32) and if that fails then try allocating 2406 2356 * from higher range 2407 2357 */ 2408 - iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32)); 2409 - if (!iova) 2410 - iova = iommu_alloc_iova(domain, size, dma_mask); 2358 + iova = alloc_iova(&domain->iovad, nrpages, 2359 + IOVA_PFN(DMA_BIT_MASK(32)), 1); 2360 + if (iova) 2361 + return iova; 2411 2362 } 2412 - 2413 - if (!iova) { 2414 - printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev)); 2363 + iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); 2364 + if (unlikely(!iova)) { 2365 + printk(KERN_ERR "Allocating %ld-page iova for %s failed", 2366 + nrpages, pci_name(pdev)); 2415 2367 return NULL; 2416 2368 } 2417 2369 ··· 2504 2476 return 0; 2505 2477 2506 2478 iommu = domain_get_iommu(domain); 2507 - size = aligned_size((u64)paddr, size); 2479 + size = aligned_nrpages(paddr, size); 2508 2480 2509 - iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask); 2481 + iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask); 2510 2482 if (!iova) 2511 2483 goto error; 2512 - 2513 - start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT; 2514 2484 2515 2485 /* 2516 2486 * Check if DMAR supports zero-length reads on write only ··· 2525 2499 * might have two guest_addr mapping to the same host paddr, but this 2526 2500 * is not a big problem 2527 2501 */ 2528 - ret = domain_page_mapping(domain, start_paddr, 2529 - ((u64)paddr) & PHYSICAL_PAGE_MASK, 2530 - size, prot); 2502 + ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo), 2503 + paddr >> VTD_PAGE_SHIFT, size, prot); 2531 2504 if (ret) 2532 2505 goto error; 2533 2506 2534 2507 /* it's a non-present to present mapping. Only flush if caching mode */ 2535 2508 if (cap_caching_mode(iommu->cap)) 2536 - iommu_flush_iotlb_psi(iommu, 0, start_paddr, 2537 - size >> VTD_PAGE_SHIFT); 2509 + iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size); 2538 2510 else 2539 2511 iommu_flush_write_buffer(iommu); 2540 2512 2541 - return start_paddr + ((u64)paddr & (~PAGE_MASK)); 2513 + start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT; 2514 + start_paddr += paddr & ~PAGE_MASK; 2515 + return start_paddr; 2542 2516 2543 2517 error: 2544 2518 if (iova) ··· 2631 2605 { 2632 2606 struct pci_dev *pdev = to_pci_dev(dev); 2633 2607 struct dmar_domain *domain; 2634 - unsigned long start_addr; 2608 + unsigned long start_pfn, last_pfn; 2635 2609 struct iova *iova; 2636 2610 struct intel_iommu *iommu; 2637 2611 ··· 2644 2618 iommu = domain_get_iommu(domain); 2645 2619 2646 2620 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr)); 2647 - if (!iova) 2621 + if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n", 2622 + (unsigned long long)dev_addr)) 2648 2623 return; 2649 2624 2650 - start_addr = iova->pfn_lo << PAGE_SHIFT; 2651 - size = aligned_size((u64)dev_addr, size); 2625 + start_pfn = mm_to_dma_pfn(iova->pfn_lo); 2626 + last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1; 2652 2627 2653 - pr_debug("Device %s unmapping: %zx@%llx\n", 2654 - pci_name(pdev), size, (unsigned long long)start_addr); 2628 + pr_debug("Device %s unmapping: pfn %lx-%lx\n", 2629 + pci_name(pdev), start_pfn, last_pfn); 2655 2630 2656 2631 /* clear the whole page */ 2657 - dma_pte_clear_range(domain, start_addr, start_addr + size); 2632 + dma_pte_clear_range(domain, start_pfn, last_pfn); 2633 + 2658 2634 /* free page tables */ 2659 - dma_pte_free_pagetable(domain, start_addr, start_addr + size); 2635 + dma_pte_free_pagetable(domain, start_pfn, last_pfn); 2636 + 2660 2637 if (intel_iommu_strict) { 2661 - iommu_flush_iotlb_psi(iommu, domain->id, start_addr, 2662 - size >> VTD_PAGE_SHIFT); 2638 + iommu_flush_iotlb_psi(iommu, domain->id, start_pfn, 2639 + last_pfn - start_pfn + 1); 2663 2640 /* free iova */ 2664 2641 __free_iova(&domain->iovad, iova); 2665 2642 } else { ··· 2720 2691 int nelems, enum dma_data_direction dir, 2721 2692 struct dma_attrs *attrs) 2722 2693 { 2723 - int i; 2724 2694 struct pci_dev *pdev = to_pci_dev(hwdev); 2725 2695 struct dmar_domain *domain; 2726 - unsigned long start_addr; 2696 + unsigned long start_pfn, last_pfn; 2727 2697 struct iova *iova; 2728 - size_t size = 0; 2729 - phys_addr_t addr; 2730 - struct scatterlist *sg; 2731 2698 struct intel_iommu *iommu; 2732 2699 2733 2700 if (iommu_no_mapping(pdev)) ··· 2735 2710 iommu = domain_get_iommu(domain); 2736 2711 2737 2712 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address)); 2738 - if (!iova) 2713 + if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n", 2714 + (unsigned long long)sglist[0].dma_address)) 2739 2715 return; 2740 - for_each_sg(sglist, sg, nelems, i) { 2741 - addr = page_to_phys(sg_page(sg)) + sg->offset; 2742 - size += aligned_size((u64)addr, sg->length); 2743 - } 2744 2716 2745 - start_addr = iova->pfn_lo << PAGE_SHIFT; 2717 + start_pfn = mm_to_dma_pfn(iova->pfn_lo); 2718 + last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1; 2746 2719 2747 2720 /* clear the whole page */ 2748 - dma_pte_clear_range(domain, start_addr, start_addr + size); 2749 - /* free page tables */ 2750 - dma_pte_free_pagetable(domain, start_addr, start_addr + size); 2721 + dma_pte_clear_range(domain, start_pfn, last_pfn); 2751 2722 2752 - iommu_flush_iotlb_psi(iommu, domain->id, start_addr, 2753 - size >> VTD_PAGE_SHIFT); 2723 + /* free page tables */ 2724 + dma_pte_free_pagetable(domain, start_pfn, last_pfn); 2725 + 2726 + iommu_flush_iotlb_psi(iommu, domain->id, start_pfn, 2727 + (last_pfn - start_pfn + 1)); 2754 2728 2755 2729 /* free iova */ 2756 2730 __free_iova(&domain->iovad, iova); ··· 2772 2748 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems, 2773 2749 enum dma_data_direction dir, struct dma_attrs *attrs) 2774 2750 { 2775 - phys_addr_t addr; 2776 2751 int i; 2777 2752 struct pci_dev *pdev = to_pci_dev(hwdev); 2778 2753 struct dmar_domain *domain; 2779 2754 size_t size = 0; 2780 2755 int prot = 0; 2781 - size_t offset = 0; 2756 + size_t offset_pfn = 0; 2782 2757 struct iova *iova = NULL; 2783 2758 int ret; 2784 2759 struct scatterlist *sg; 2785 - unsigned long start_addr; 2760 + unsigned long start_vpfn; 2786 2761 struct intel_iommu *iommu; 2787 2762 2788 2763 BUG_ON(dir == DMA_NONE); ··· 2794 2771 2795 2772 iommu = domain_get_iommu(domain); 2796 2773 2797 - for_each_sg(sglist, sg, nelems, i) { 2798 - addr = page_to_phys(sg_page(sg)) + sg->offset; 2799 - size += aligned_size((u64)addr, sg->length); 2800 - } 2774 + for_each_sg(sglist, sg, nelems, i) 2775 + size += aligned_nrpages(sg->offset, sg->length); 2801 2776 2802 - iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask); 2777 + iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask); 2803 2778 if (!iova) { 2804 2779 sglist->dma_length = 0; 2805 2780 return 0; ··· 2813 2792 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) 2814 2793 prot |= DMA_PTE_WRITE; 2815 2794 2816 - start_addr = iova->pfn_lo << PAGE_SHIFT; 2817 - offset = 0; 2818 - for_each_sg(sglist, sg, nelems, i) { 2819 - addr = page_to_phys(sg_page(sg)) + sg->offset; 2820 - size = aligned_size((u64)addr, sg->length); 2821 - ret = domain_page_mapping(domain, start_addr + offset, 2822 - ((u64)addr) & PHYSICAL_PAGE_MASK, 2823 - size, prot); 2824 - if (ret) { 2825 - /* clear the page */ 2826 - dma_pte_clear_range(domain, start_addr, 2827 - start_addr + offset); 2828 - /* free page tables */ 2829 - dma_pte_free_pagetable(domain, start_addr, 2830 - start_addr + offset); 2831 - /* free iova */ 2832 - __free_iova(&domain->iovad, iova); 2833 - return 0; 2834 - } 2835 - sg->dma_address = start_addr + offset + 2836 - ((u64)addr & (~PAGE_MASK)); 2837 - sg->dma_length = sg->length; 2838 - offset += size; 2795 + start_vpfn = mm_to_dma_pfn(iova->pfn_lo); 2796 + 2797 + ret = domain_sg_mapping(domain, start_vpfn, sglist, mm_to_dma_pfn(size), prot); 2798 + if (unlikely(ret)) { 2799 + /* clear the page */ 2800 + dma_pte_clear_range(domain, start_vpfn, 2801 + start_vpfn + size - 1); 2802 + /* free page tables */ 2803 + dma_pte_free_pagetable(domain, start_vpfn, 2804 + start_vpfn + size - 1); 2805 + /* free iova */ 2806 + __free_iova(&domain->iovad, iova); 2807 + return 0; 2839 2808 } 2840 2809 2841 2810 /* it's a non-present to present mapping. Only flush if caching mode */ 2842 2811 if (cap_caching_mode(iommu->cap)) 2843 - iommu_flush_iotlb_psi(iommu, 0, start_addr, 2844 - offset >> VTD_PAGE_SHIFT); 2812 + iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn); 2845 2813 else 2846 2814 iommu_flush_write_buffer(iommu); 2847 2815 ··· 3335 3325 int adjust_width; 3336 3326 3337 3327 init_iova_domain(&domain->iovad, DMA_32BIT_PFN); 3338 - spin_lock_init(&domain->mapping_lock); 3339 3328 spin_lock_init(&domain->iommu_lock); 3340 3329 3341 3330 domain_reserve_special_ranges(domain); ··· 3388 3379 3389 3380 static void vm_domain_exit(struct dmar_domain *domain) 3390 3381 { 3391 - u64 end; 3392 - 3393 3382 /* Domain 0 is reserved, so dont process it */ 3394 3383 if (!domain) 3395 3384 return; ··· 3395 3388 vm_domain_remove_all_dev_info(domain); 3396 3389 /* destroy iovas */ 3397 3390 put_iova_domain(&domain->iovad); 3398 - end = DOMAIN_MAX_ADDR(domain->gaw); 3399 - end = end & (~VTD_PAGE_MASK); 3400 3391 3401 3392 /* clear ptes */ 3402 - dma_pte_clear_range(domain, 0, end); 3393 + dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); 3403 3394 3404 3395 /* free page tables */ 3405 - dma_pte_free_pagetable(domain, 0, end); 3396 + dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); 3406 3397 3407 3398 iommu_free_vm_domain(domain); 3408 3399 free_domain_mem(domain); ··· 3509 3504 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) 3510 3505 prot |= DMA_PTE_SNP; 3511 3506 3512 - max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size); 3507 + max_addr = iova + size; 3513 3508 if (dmar_domain->max_addr < max_addr) { 3514 3509 int min_agaw; 3515 3510 u64 end; ··· 3527 3522 } 3528 3523 dmar_domain->max_addr = max_addr; 3529 3524 } 3530 - 3531 - ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot); 3525 + /* Round up size to next multiple of PAGE_SIZE, if it and 3526 + the low bits of hpa would take us onto the next page */ 3527 + size = aligned_nrpages(hpa, size); 3528 + ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT, 3529 + hpa >> VTD_PAGE_SHIFT, size, prot); 3532 3530 return ret; 3533 3531 } 3534 3532 ··· 3539 3531 unsigned long iova, size_t size) 3540 3532 { 3541 3533 struct dmar_domain *dmar_domain = domain->priv; 3542 - dma_addr_t base; 3543 3534 3544 - /* The address might not be aligned */ 3545 - base = iova & VTD_PAGE_MASK; 3546 - size = VTD_PAGE_ALIGN(size); 3547 - dma_pte_clear_range(dmar_domain, base, base + size); 3535 + dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, 3536 + (iova + size - 1) >> VTD_PAGE_SHIFT); 3548 3537 3549 - if (dmar_domain->max_addr == base + size) 3550 - dmar_domain->max_addr = base; 3538 + if (dmar_domain->max_addr == iova + size) 3539 + dmar_domain->max_addr = iova; 3551 3540 } 3552 3541 3553 3542 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, ··· 3554 3549 struct dma_pte *pte; 3555 3550 u64 phys = 0; 3556 3551 3557 - pte = addr_to_dma_pte(dmar_domain, iova); 3552 + pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT); 3558 3553 if (pte) 3559 3554 phys = dma_pte_addr(pte); 3560 3555

Configure Feed

Configure Feed