Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommu/io-pgtable-dart: Add 4-level page table support
DARTs on t602x SoCs are of the t8110 variant but have an IAS of 42,
which means optional support for an extra page table level.
Refactor the PTE management to support an arbitrary level count, and
then calculate how many levels we need for any given configuration.
Signed-off-by: Hector Martin <marcan@marcan.st>
Signed-off-by: Janne Grunau <j@jannau.net>
Reviewed-by: Sven Peter <sven@kernel.org>
Reviewed-by: Neal Gompa <neal@gompa.dev>
Link: https://lore.kernel.org/r/20250821-apple-dart-4levels-v2-2-e39af79daa37@jannau.net
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
authored by
Hector Martin
and committed by
Joerg Roedel
74a0e72f
12688900
+89
-55
+88
-55
drivers/iommu/io-pgtable-dart.c
+88
-55
drivers/iommu/io-pgtable-dart.c
···
27
27
28
28
#define DART1_MAX_ADDR_BITS 36
29
29
30
-
#define DART_MAX_TABLES 4
31
-
#define DART_LEVELS 2
30
+
#define DART_MAX_TABLE_BITS 2
31
+
#define DART_MAX_TABLES BIT(DART_MAX_TABLE_BITS)
32
+
#define DART_MAX_LEVELS 4 /* Includes TTBR level */
32
33
33
34
/* Struct accessors */
34
35
#define io_pgtable_to_data(x) \
···
69
68
struct dart_io_pgtable {
70
69
struct io_pgtable iop;
71
70
71
+
int levels;
72
72
int tbl_bits;
73
73
int bits_per_level;
74
74
···
158
156
return old;
159
157
}
160
158
161
-
static int dart_get_table(struct dart_io_pgtable *data, unsigned long iova)
159
+
static int dart_get_index(struct dart_io_pgtable *data, unsigned long iova, int level)
162
160
{
163
-
return (iova >> (3 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) &
164
-
((1 << data->tbl_bits) - 1);
161
+
return (iova >> (level * data->bits_per_level + ilog2(sizeof(dart_iopte)))) &
162
+
((1 << data->bits_per_level) - 1);
165
163
}
166
164
167
-
static int dart_get_l1_index(struct dart_io_pgtable *data, unsigned long iova)
168
-
{
169
-
170
-
return (iova >> (2 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) &
171
-
((1 << data->bits_per_level) - 1);
172
-
}
173
-
174
-
static int dart_get_l2_index(struct dart_io_pgtable *data, unsigned long iova)
165
+
static int dart_get_last_index(struct dart_io_pgtable *data, unsigned long iova)
175
166
{
176
167
177
168
return (iova >> (data->bits_per_level + ilog2(sizeof(dart_iopte)))) &
178
169
((1 << data->bits_per_level) - 1);
179
170
}
180
171
181
-
static dart_iopte *dart_get_l2(struct dart_io_pgtable *data, unsigned long iova)
172
+
static dart_iopte *dart_get_last(struct dart_io_pgtable *data, unsigned long iova)
182
173
{
183
174
dart_iopte pte, *ptep;
184
-
int tbl = dart_get_table(data, iova);
175
+
int level = data->levels;
176
+
int tbl = dart_get_index(data, iova, level);
177
+
178
+
if (tbl > (1 << data->tbl_bits))
179
+
return NULL;
185
180
186
181
ptep = data->pgd[tbl];
187
182
if (!ptep)
188
183
return NULL;
189
184
190
-
ptep += dart_get_l1_index(data, iova);
191
-
pte = READ_ONCE(*ptep);
185
+
while (--level > 1) {
186
+
ptep += dart_get_index(data, iova, level);
187
+
pte = READ_ONCE(*ptep);
192
188
193
-
/* Valid entry? */
194
-
if (!pte)
195
-
return NULL;
189
+
/* Valid entry? */
190
+
if (!pte)
191
+
return NULL;
196
192
197
-
/* Deref to get level 2 table */
198
-
return iopte_deref(pte, data);
193
+
/* Deref to get next level table */
194
+
ptep = iopte_deref(pte, data);
195
+
}
196
+
197
+
return ptep;
199
198
}
200
199
201
200
static dart_iopte dart_prot_to_pte(struct dart_io_pgtable *data,
···
233
230
int ret = 0, tbl, num_entries, max_entries, map_idx_start;
234
231
dart_iopte pte, *cptep, *ptep;
235
232
dart_iopte prot;
233
+
int level = data->levels;
236
234
237
235
if (WARN_ON(pgsize != cfg->pgsize_bitmap))
238
236
return -EINVAL;
···
244
240
if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
245
241
return -EINVAL;
246
242
247
-
tbl = dart_get_table(data, iova);
243
+
tbl = dart_get_index(data, iova, level);
244
+
245
+
if (tbl > (1 << data->tbl_bits))
246
+
return -ENOMEM;
248
247
249
248
ptep = data->pgd[tbl];
250
-
ptep += dart_get_l1_index(data, iova);
251
-
pte = READ_ONCE(*ptep);
252
-
253
-
/* no L2 table present */
254
-
if (!pte) {
255
-
cptep = iommu_alloc_pages_sz(gfp, tblsz);
256
-
if (!cptep)
257
-
return -ENOMEM;
258
-
259
-
pte = dart_install_table(cptep, ptep, 0, data);
260
-
if (pte)
261
-
iommu_free_pages(cptep);
262
-
263
-
/* L2 table is present (now) */
249
+
while (--level > 1) {
250
+
ptep += dart_get_index(data, iova, level);
264
251
pte = READ_ONCE(*ptep);
265
-
}
266
252
267
-
ptep = iopte_deref(pte, data);
253
+
/* no table present */
254
+
if (!pte) {
255
+
cptep = iommu_alloc_pages_sz(gfp, tblsz);
256
+
if (!cptep)
257
+
return -ENOMEM;
258
+
259
+
pte = dart_install_table(cptep, ptep, 0, data);
260
+
if (pte)
261
+
iommu_free_pages(cptep);
262
+
263
+
/* L2 table is present (now) */
264
+
pte = READ_ONCE(*ptep);
265
+
}
266
+
267
+
ptep = iopte_deref(pte, data);
268
+
}
268
269
269
270
/* install a leaf entries into L2 table */
270
271
prot = dart_prot_to_pte(data, iommu_prot);
271
-
map_idx_start = dart_get_l2_index(data, iova);
272
+
map_idx_start = dart_get_last_index(data, iova);
272
273
max_entries = DART_PTES_PER_TABLE(data) - map_idx_start;
273
274
num_entries = min_t(int, pgcount, max_entries);
274
275
ptep += map_idx_start;
···
302
293
if (WARN_ON(pgsize != cfg->pgsize_bitmap || !pgcount))
303
294
return 0;
304
295
305
-
ptep = dart_get_l2(data, iova);
296
+
ptep = dart_get_last(data, iova);
306
297
307
298
/* Valid L2 IOPTE pointer? */
308
299
if (WARN_ON(!ptep))
309
300
return 0;
310
301
311
-
unmap_idx_start = dart_get_l2_index(data, iova);
302
+
unmap_idx_start = dart_get_last_index(data, iova);
312
303
ptep += unmap_idx_start;
313
304
314
305
max_entries = DART_PTES_PER_TABLE(data) - unmap_idx_start;
···
339
330
struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops);
340
331
dart_iopte pte, *ptep;
341
332
342
-
ptep = dart_get_l2(data, iova);
333
+
ptep = dart_get_last(data, iova);
343
334
344
335
/* Valid L2 IOPTE pointer? */
345
336
if (!ptep)
346
337
return 0;
347
338
348
-
ptep += dart_get_l2_index(data, iova);
339
+
ptep += dart_get_last_index(data, iova);
349
340
350
341
pte = READ_ONCE(*ptep);
351
342
/* Found translation */
···
362
353
dart_alloc_pgtable(struct io_pgtable_cfg *cfg)
363
354
{
364
355
struct dart_io_pgtable *data;
365
-
int tbl_bits, bits_per_level, va_bits, pg_shift;
356
+
int levels, max_tbl_bits, tbl_bits, bits_per_level, va_bits, pg_shift;
357
+
358
+
/*
359
+
* Old 4K page DARTs can use up to 4 top-level tables.
360
+
* Newer ones only ever use a maximum of 1.
361
+
*/
362
+
if (cfg->pgsize_bitmap == SZ_4K)
363
+
max_tbl_bits = DART_MAX_TABLE_BITS;
364
+
else
365
+
max_tbl_bits = 0;
366
366
367
367
pg_shift = __ffs(cfg->pgsize_bitmap);
368
368
bits_per_level = pg_shift - ilog2(sizeof(dart_iopte));
369
369
370
370
va_bits = cfg->ias - pg_shift;
371
371
372
-
tbl_bits = max_t(int, 0, va_bits - (bits_per_level * DART_LEVELS));
373
-
if ((1 << tbl_bits) > DART_MAX_TABLES)
372
+
levels = max_t(int, 2, (va_bits - max_tbl_bits + bits_per_level - 1) / bits_per_level);
373
+
374
+
if (levels > (DART_MAX_LEVELS - 1))
375
+
return NULL;
376
+
377
+
tbl_bits = max_t(int, 0, va_bits - (bits_per_level * levels));
378
+
379
+
if (tbl_bits > max_tbl_bits)
374
380
return NULL;
375
381
376
382
data = kzalloc(sizeof(*data), GFP_KERNEL);
377
383
if (!data)
378
384
return NULL;
379
385
386
+
data->levels = levels + 1; /* Table level counts as one level */
380
387
data->tbl_bits = tbl_bits;
381
388
data->bits_per_level = bits_per_level;
382
389
···
428
403
return NULL;
429
404
430
405
cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits;
406
+
cfg->apple_dart_cfg.n_levels = data->levels;
431
407
432
408
for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) {
433
409
data->pgd[i] =
···
448
422
return NULL;
449
423
}
450
424
451
-
static void apple_dart_free_pgtable(struct io_pgtable *iop)
425
+
static void apple_dart_free_pgtables(struct dart_io_pgtable *data, dart_iopte *ptep, int level)
452
426
{
453
-
struct dart_io_pgtable *data = io_pgtable_to_data(iop);
454
-
dart_iopte *ptep, *end;
455
-
int i;
427
+
dart_iopte *end;
428
+
dart_iopte *start = ptep;
456
429
457
-
for (i = 0; i < (1 << data->tbl_bits) && data->pgd[i]; ++i) {
458
-
ptep = data->pgd[i];
430
+
if (level > 1) {
459
431
end = (void *)ptep + DART_GRANULE(data);
460
432
461
433
while (ptep != end) {
462
434
dart_iopte pte = *ptep++;
463
435
464
436
if (pte)
465
-
iommu_free_pages(iopte_deref(pte, data));
437
+
apple_dart_free_pgtables(data, iopte_deref(pte, data), level - 1);
466
438
}
467
-
iommu_free_pages(data->pgd[i]);
468
439
}
440
+
iommu_free_pages(start);
441
+
}
442
+
443
+
static void apple_dart_free_pgtable(struct io_pgtable *iop)
444
+
{
445
+
struct dart_io_pgtable *data = io_pgtable_to_data(iop);
446
+
int i;
447
+
448
+
for (i = 0; i < (1 << data->tbl_bits) && data->pgd[i]; ++i)
449
+
apple_dart_free_pgtables(data, data->pgd[i], data->levels - 1);
469
450
470
451
kfree(data);
471
452
}