Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'iommu-fixes-v3.14-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU fixes from Joerg Roedel:
"The fixes are only for the ARM-SMMU driver. Here is the summary from
Will Deacon:
- Andreas Herrmann took the driver for a run with a real SATA
controller, which caused the new mutex-based locking to explode
since we require mappings in atomic context
- Yifan fixed an issue with the page table creation, which then
caused breakages with the way in which we flush descriptors out to
the table walker
- I ran the driver on a system where the SMMU is hooked into a
coherent interconnect for table walks, and noticed a shareability
mismatch between the CPU and the SMMU
These issues are all fixed here and have been tested on both arm and
arm64 based systems.
Besides that I put a fix on-top to make the spinlock irq-safe, so that
the code-paths can be used in the DMA-API"
* tag 'iommu-fixes-v3.14-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu:
arm/smmu: Use irqsafe spinlock for domain lock
iommu/arm-smmu: fix compilation issue when !CONFIG_ARM_AMBA
iommu/arm-smmu: set CBARn.BPSHCFG to NSH for s1-s2-bypass contexts
iommu/arm-smmu: fix table flushing during initial allocations
iommu/arm-smmu: really fix page table locking
iommu/arm-smmu: fix pud/pmd entry fill sequence
+63
-42
+63
-42
drivers/iommu/arm-smmu.c
+63
-42
drivers/iommu/arm-smmu.c
···
79
79
80
80
#define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES)
81
81
#define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1))
82
-
#define ARM_SMMU_PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(pte_t))
83
82
84
83
/* Stage-1 PTE */
85
84
#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6)
···
190
191
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
191
192
#define CBAR_VMID_SHIFT 0
192
193
#define CBAR_VMID_MASK 0xff
194
+
#define CBAR_S1_BPSHCFG_SHIFT 8
195
+
#define CBAR_S1_BPSHCFG_MASK 3
196
+
#define CBAR_S1_BPSHCFG_NSH 3
193
197
#define CBAR_S1_MEMATTR_SHIFT 12
194
198
#define CBAR_S1_MEMATTR_MASK 0xf
195
199
#define CBAR_S1_MEMATTR_WB 0xf
···
395
393
struct arm_smmu_cfg root_cfg;
396
394
phys_addr_t output_mask;
397
395
398
-
struct mutex lock;
396
+
spinlock_t lock;
399
397
};
400
398
401
399
static DEFINE_SPINLOCK(arm_smmu_devices_lock);
···
634
632
return IRQ_HANDLED;
635
633
}
636
634
635
+
static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr,
636
+
size_t size)
637
+
{
638
+
unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
639
+
640
+
641
+
/* Ensure new page tables are visible to the hardware walker */
642
+
if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) {
643
+
dsb();
644
+
} else {
645
+
/*
646
+
* If the SMMU can't walk tables in the CPU caches, treat them
647
+
* like non-coherent DMA since we need to flush the new entries
648
+
* all the way out to memory. There's no possibility of
649
+
* recursion here as the SMMU table walker will not be wired
650
+
* through another SMMU.
651
+
*/
652
+
dma_map_page(smmu->dev, virt_to_page(addr), offset, size,
653
+
DMA_TO_DEVICE);
654
+
}
655
+
}
656
+
637
657
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
638
658
{
639
659
u32 reg;
···
674
650
if (smmu->version == 1)
675
651
reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
676
652
677
-
/* Use the weakest memory type, so it is overridden by the pte */
678
-
if (stage1)
679
-
reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
680
-
else
653
+
/*
654
+
* Use the weakest shareability/memory types, so they are
655
+
* overridden by the ttbcr/pte.
656
+
*/
657
+
if (stage1) {
658
+
reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
659
+
(CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
660
+
} else {
681
661
reg |= ARM_SMMU_CB_VMID(root_cfg) << CBAR_VMID_SHIFT;
662
+
}
682
663
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
683
664
684
665
if (smmu->version > 1) {
···
744
715
}
745
716
746
717
/* TTBR0 */
718
+
arm_smmu_flush_pgtable(smmu, root_cfg->pgd,
719
+
PTRS_PER_PGD * sizeof(pgd_t));
747
720
reg = __pa(root_cfg->pgd);
748
721
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
749
722
reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
···
932
901
goto out_free_domain;
933
902
smmu_domain->root_cfg.pgd = pgd;
934
903
935
-
mutex_init(&smmu_domain->lock);
904
+
spin_lock_init(&smmu_domain->lock);
936
905
domain->priv = smmu_domain;
937
906
return 0;
938
907
···
1159
1128
struct arm_smmu_domain *smmu_domain = domain->priv;
1160
1129
struct arm_smmu_device *device_smmu = dev->archdata.iommu;
1161
1130
struct arm_smmu_master *master;
1131
+
unsigned long flags;
1162
1132
1163
1133
if (!device_smmu) {
1164
1134
dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
···
1170
1138
* Sanity check the domain. We don't currently support domains
1171
1139
* that cross between different SMMU chains.
1172
1140
*/
1173
-
mutex_lock(&smmu_domain->lock);
1141
+
spin_lock_irqsave(&smmu_domain->lock, flags);
1174
1142
if (!smmu_domain->leaf_smmu) {
1175
1143
/* Now that we have a master, we can finalise the domain */
1176
1144
ret = arm_smmu_init_domain_context(domain, dev);
···
1185
1153
dev_name(device_smmu->dev));
1186
1154
goto err_unlock;
1187
1155
}
1188
-
mutex_unlock(&smmu_domain->lock);
1156
+
spin_unlock_irqrestore(&smmu_domain->lock, flags);
1189
1157
1190
1158
/* Looks ok, so add the device to the domain */
1191
1159
master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node);
···
1195
1163
return arm_smmu_domain_add_master(smmu_domain, master);
1196
1164
1197
1165
err_unlock:
1198
-
mutex_unlock(&smmu_domain->lock);
1166
+
spin_unlock_irqrestore(&smmu_domain->lock, flags);
1199
1167
return ret;
1200
1168
}
1201
1169
···
1207
1175
master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node);
1208
1176
if (master)
1209
1177
arm_smmu_domain_remove_master(smmu_domain, master);
1210
-
}
1211
-
1212
-
static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr,
1213
-
size_t size)
1214
-
{
1215
-
unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
1216
-
1217
-
/*
1218
-
* If the SMMU can't walk tables in the CPU caches, treat them
1219
-
* like non-coherent DMA since we need to flush the new entries
1220
-
* all the way out to memory. There's no possibility of recursion
1221
-
* here as the SMMU table walker will not be wired through another
1222
-
* SMMU.
1223
-
*/
1224
-
if (!(smmu->features & ARM_SMMU_FEAT_COHERENT_WALK))
1225
-
dma_map_page(smmu->dev, virt_to_page(addr), offset, size,
1226
-
DMA_TO_DEVICE);
1227
1178
}
1228
1179
1229
1180
static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
···
1225
1210
1226
1211
if (pmd_none(*pmd)) {
1227
1212
/* Allocate a new set of tables */
1228
-
pgtable_t table = alloc_page(PGALLOC_GFP);
1213
+
pgtable_t table = alloc_page(GFP_ATOMIC|__GFP_ZERO);
1229
1214
if (!table)
1230
1215
return -ENOMEM;
1231
1216
1232
-
arm_smmu_flush_pgtable(smmu, page_address(table),
1233
-
ARM_SMMU_PTE_HWTABLE_SIZE);
1217
+
arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE);
1234
1218
if (!pgtable_page_ctor(table)) {
1235
1219
__free_page(table);
1236
1220
return -ENOMEM;
···
1331
1317
1332
1318
#ifndef __PAGETABLE_PMD_FOLDED
1333
1319
if (pud_none(*pud)) {
1334
-
pmd = pmd_alloc_one(NULL, addr);
1320
+
pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC);
1335
1321
if (!pmd)
1336
1322
return -ENOMEM;
1323
+
1324
+
arm_smmu_flush_pgtable(smmu, pmd, PAGE_SIZE);
1325
+
pud_populate(NULL, pud, pmd);
1326
+
arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud));
1327
+
1328
+
pmd += pmd_index(addr);
1337
1329
} else
1338
1330
#endif
1339
1331
pmd = pmd_offset(pud, addr);
···
1348
1328
next = pmd_addr_end(addr, end);
1349
1329
ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, end, pfn,
1350
1330
flags, stage);
1351
-
pud_populate(NULL, pud, pmd);
1352
-
arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud));
1353
1331
phys += next - addr;
1354
1332
} while (pmd++, addr = next, addr < end);
1355
1333
···
1364
1346
1365
1347
#ifndef __PAGETABLE_PUD_FOLDED
1366
1348
if (pgd_none(*pgd)) {
1367
-
pud = pud_alloc_one(NULL, addr);
1349
+
pud = (pud_t *)get_zeroed_page(GFP_ATOMIC);
1368
1350
if (!pud)
1369
1351
return -ENOMEM;
1352
+
1353
+
arm_smmu_flush_pgtable(smmu, pud, PAGE_SIZE);
1354
+
pgd_populate(NULL, pgd, pud);
1355
+
arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd));
1356
+
1357
+
pud += pud_index(addr);
1370
1358
} else
1371
1359
#endif
1372
1360
pud = pud_offset(pgd, addr);
···
1381
1357
next = pud_addr_end(addr, end);
1382
1358
ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys,
1383
1359
flags, stage);
1384
-
pgd_populate(NULL, pud, pgd);
1385
-
arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd));
1386
1360
phys += next - addr;
1387
1361
} while (pud++, addr = next, addr < end);
1388
1362
···
1397
1375
struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
1398
1376
pgd_t *pgd = root_cfg->pgd;
1399
1377
struct arm_smmu_device *smmu = root_cfg->smmu;
1378
+
unsigned long irqflags;
1400
1379
1401
1380
if (root_cfg->cbar == CBAR_TYPE_S2_TRANS) {
1402
1381
stage = 2;
···
1420
1397
if (paddr & ~output_mask)
1421
1398
return -ERANGE;
1422
1399
1423
-
mutex_lock(&smmu_domain->lock);
1400
+
spin_lock_irqsave(&smmu_domain->lock, irqflags);
1424
1401
pgd += pgd_index(iova);
1425
1402
end = iova + size;
1426
1403
do {
···
1436
1413
} while (pgd++, iova != end);
1437
1414
1438
1415
out_unlock:
1439
-
mutex_unlock(&smmu_domain->lock);
1440
-
1441
-
/* Ensure new page tables are visible to the hardware walker */
1442
-
if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
1443
-
dsb();
1416
+
spin_unlock_irqrestore(&smmu_domain->lock, irqflags);
1444
1417
1445
1418
return ret;
1446
1419
}
···
2006
1987
if (!iommu_present(&platform_bus_type))
2007
1988
bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2008
1989
1990
+
#ifdef CONFIG_ARM_AMBA
2009
1991
if (!iommu_present(&amba_bustype))
2010
1992
bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1993
+
#endif
2011
1994
2012
1995
return 0;
2013
1996
}