Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
drm/amdgpu: add drm buddy support to amdgpu
- Switch to drm buddy allocator
- Add resource cursor support for drm buddy
v2(Matthew Auld):
- replace spinlock with mutex as we call kmem_cache_zalloc
(..., GFP_KERNEL) in drm_buddy_alloc() function
- lock drm_buddy_block_trim() function as it calls
mark_free/mark_split are all globally visible
v3(Matthew Auld):
- remove trim method error handling as we address the failure case
at drm_buddy_block_trim() function
v4:
- fix warnings reported by kernel test robot <lkp@intel.com>
v5:
- fix merge conflict issue
v6:
- fix warnings reported by kernel test robot <lkp@intel.com>
v7:
- remove DRM_BUDDY_RANGE_ALLOCATION flag usage
v8:
- keep DRM_BUDDY_RANGE_ALLOCATION flag usage
- resolve conflicts created by drm/amdgpu: remove VRAM accounting v2
v9(Christian):
- merged the below patch
- drm/amdgpu: move vram inline functions into a header
- rename label name as fallback
- move struct amdgpu_vram_mgr to amdgpu_vram_mgr.h
- remove unnecessary flags from struct amdgpu_vram_reservation
- rewrite block NULL check condition
- change else style as per coding standard
- rewrite the node max size
- add a helper function to fetch the first entry from the list
v10(Christian):
- rename amdgpu_get_node() function name as amdgpu_vram_mgr_first_block
v11:
- if size is not aligned with min_page_size, enable is_contiguous flag,
therefore, the size round up to the power of two and trimmed to the
original size.
v12:
- rename the function names having prefix as amdgpu_vram_mgr_*()
- modify the round_up() logic conforming to contiguous flag enablement
or if size is not aligned to min_block_size
- modify the trim logic
- rename node as block wherever applicable
Signed-off-by: Arunpravin Paneer Selvam <Arunpravin.PaneerSelvam@amd.com>
Acked-by: Christian König <christian.koenig@amd.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20220407224843.2416-1-Arunpravin.PaneerSelvam@amd.com
Signed-off-by: Christian König <christian.koenig@amd.com>
authored by
Arunpravin Paneer Selvam
and committed by
Christian König
c9cad937
17b048d4
+375
-171
+1
drivers/gpu/drm/Kconfig
+1
drivers/gpu/drm/Kconfig
+78
-19
drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+78
-19
drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
···
30
30
#include <drm/ttm/ttm_resource.h>
31
31
#include <drm/ttm/ttm_range_manager.h>
32
32
33
+
#include "amdgpu_vram_mgr.h"
34
+
33
35
/* state back for walking over vram_mgr and gtt_mgr allocations */
34
36
struct amdgpu_res_cursor {
35
37
uint64_t start;
36
38
uint64_t size;
37
39
uint64_t remaining;
38
-
struct drm_mm_node *node;
40
+
void *node;
41
+
uint32_t mem_type;
39
42
};
40
43
41
44
/**
···
55
52
uint64_t start, uint64_t size,
56
53
struct amdgpu_res_cursor *cur)
57
54
{
55
+
struct drm_buddy_block *block;
56
+
struct list_head *head, *next;
58
57
struct drm_mm_node *node;
59
58
60
-
if (!res || res->mem_type == TTM_PL_SYSTEM) {
61
-
cur->start = start;
62
-
cur->size = size;
63
-
cur->remaining = size;
64
-
cur->node = NULL;
65
-
WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT);
66
-
return;
67
-
}
59
+
if (!res)
60
+
goto fallback;
68
61
69
62
BUG_ON(start + size > res->num_pages << PAGE_SHIFT);
70
63
71
-
node = to_ttm_range_mgr_node(res)->mm_nodes;
72
-
while (start >= node->size << PAGE_SHIFT)
73
-
start -= node++->size << PAGE_SHIFT;
64
+
cur->mem_type = res->mem_type;
74
65
75
-
cur->start = (node->start << PAGE_SHIFT) + start;
76
-
cur->size = min((node->size << PAGE_SHIFT) - start, size);
66
+
switch (cur->mem_type) {
67
+
case TTM_PL_VRAM:
68
+
head = &to_amdgpu_vram_mgr_resource(res)->blocks;
69
+
70
+
block = list_first_entry_or_null(head,
71
+
struct drm_buddy_block,
72
+
link);
73
+
if (!block)
74
+
goto fallback;
75
+
76
+
while (start >= amdgpu_vram_mgr_block_size(block)) {
77
+
start -= amdgpu_vram_mgr_block_size(block);
78
+
79
+
next = block->link.next;
80
+
if (next != head)
81
+
block = list_entry(next, struct drm_buddy_block, link);
82
+
}
83
+
84
+
cur->start = amdgpu_vram_mgr_block_start(block) + start;
85
+
cur->size = min(amdgpu_vram_mgr_block_size(block) - start, size);
86
+
cur->remaining = size;
87
+
cur->node = block;
88
+
break;
89
+
case TTM_PL_TT:
90
+
node = to_ttm_range_mgr_node(res)->mm_nodes;
91
+
while (start >= node->size << PAGE_SHIFT)
92
+
start -= node++->size << PAGE_SHIFT;
93
+
94
+
cur->start = (node->start << PAGE_SHIFT) + start;
95
+
cur->size = min((node->size << PAGE_SHIFT) - start, size);
96
+
cur->remaining = size;
97
+
cur->node = node;
98
+
break;
99
+
default:
100
+
goto fallback;
101
+
}
102
+
103
+
return;
104
+
105
+
fallback:
106
+
cur->start = start;
107
+
cur->size = size;
77
108
cur->remaining = size;
78
-
cur->node = node;
109
+
cur->node = NULL;
110
+
WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT);
111
+
return;
79
112
}
80
113
81
114
/**
···
124
85
*/
125
86
static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
126
87
{
127
-
struct drm_mm_node *node = cur->node;
88
+
struct drm_buddy_block *block;
89
+
struct drm_mm_node *node;
90
+
struct list_head *next;
128
91
129
92
BUG_ON(size > cur->remaining);
130
93
···
140
99
return;
141
100
}
142
101
143
-
cur->node = ++node;
144
-
cur->start = node->start << PAGE_SHIFT;
145
-
cur->size = min(node->size << PAGE_SHIFT, cur->remaining);
102
+
switch (cur->mem_type) {
103
+
case TTM_PL_VRAM:
104
+
block = cur->node;
105
+
106
+
next = block->link.next;
107
+
block = list_entry(next, struct drm_buddy_block, link);
108
+
109
+
cur->node = block;
110
+
cur->start = amdgpu_vram_mgr_block_start(block);
111
+
cur->size = min(amdgpu_vram_mgr_block_size(block), cur->remaining);
112
+
break;
113
+
case TTM_PL_TT:
114
+
node = cur->node;
115
+
116
+
cur->node = ++node;
117
+
cur->start = node->start << PAGE_SHIFT;
118
+
cur->size = min(node->size << PAGE_SHIFT, cur->remaining);
119
+
break;
120
+
default:
121
+
return;
122
+
}
146
123
}
147
124
148
125
#endif
+1
-9
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+1
-9
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
···
26
26
27
27
#include <linux/dma-direction.h>
28
28
#include <drm/gpu_scheduler.h>
29
+
#include "amdgpu_vram_mgr.h"
29
30
#include "amdgpu.h"
30
31
31
32
#define AMDGPU_PL_GDS (TTM_PL_PRIV + 0)
···
38
37
#define AMDGPU_GTT_NUM_TRANSFER_WINDOWS 2
39
38
40
39
#define AMDGPU_POISON 0xd0bed0be
41
-
42
-
struct amdgpu_vram_mgr {
43
-
struct ttm_resource_manager manager;
44
-
struct drm_mm mm;
45
-
spinlock_t lock;
46
-
struct list_head reservations_pending;
47
-
struct list_head reserved_pages;
48
-
atomic64_t vis_usage;
49
-
};
50
40
51
41
struct amdgpu_gtt_mgr {
52
42
struct ttm_resource_manager manager;
+206
-143
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+206
-143
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
···
32
32
#include "atom.h"
33
33
34
34
struct amdgpu_vram_reservation {
35
-
struct list_head node;
36
-
struct drm_mm_node mm_node;
35
+
u64 start;
36
+
u64 size;
37
+
struct list_head allocated;
38
+
struct list_head blocks;
37
39
};
38
40
39
41
static inline struct amdgpu_vram_mgr *
···
188
186
};
189
187
190
188
/**
191
-
* amdgpu_vram_mgr_vis_size - Calculate visible node size
189
+
* amdgpu_vram_mgr_vis_size - Calculate visible block size
192
190
*
193
191
* @adev: amdgpu_device pointer
194
-
* @node: MM node structure
192
+
* @block: DRM BUDDY block structure
195
193
*
196
-
* Calculate how many bytes of the MM node are inside visible VRAM
194
+
* Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
197
195
*/
198
196
static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
199
-
struct drm_mm_node *node)
197
+
struct drm_buddy_block *block)
200
198
{
201
-
uint64_t start = node->start << PAGE_SHIFT;
202
-
uint64_t end = (node->size + node->start) << PAGE_SHIFT;
199
+
u64 start = amdgpu_vram_mgr_block_start(block);
200
+
u64 end = start + amdgpu_vram_mgr_block_size(block);
203
201
204
202
if (start >= adev->gmc.visible_vram_size)
205
203
return 0;
···
220
218
{
221
219
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
222
220
struct ttm_resource *res = bo->tbo.resource;
223
-
unsigned pages = res->num_pages;
224
-
struct drm_mm_node *mm;
225
-
u64 usage;
221
+
struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
222
+
struct drm_buddy_block *block;
223
+
u64 usage = 0;
226
224
227
225
if (amdgpu_gmc_vram_full_visible(&adev->gmc))
228
226
return amdgpu_bo_size(bo);
···
230
228
if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
231
229
return 0;
232
230
233
-
mm = &container_of(res, struct ttm_range_mgr_node, base)->mm_nodes[0];
234
-
for (usage = 0; pages; pages -= mm->size, mm++)
235
-
usage += amdgpu_vram_mgr_vis_size(adev, mm);
231
+
list_for_each_entry(block, &vres->blocks, link)
232
+
usage += amdgpu_vram_mgr_vis_size(adev, block);
236
233
237
234
return usage;
238
235
}
···
241
240
{
242
241
struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
243
242
struct amdgpu_device *adev = to_amdgpu_device(mgr);
244
-
struct drm_mm *mm = &mgr->mm;
243
+
struct drm_buddy *mm = &mgr->mm;
245
244
struct amdgpu_vram_reservation *rsv, *temp;
245
+
struct drm_buddy_block *block;
246
246
uint64_t vis_usage;
247
247
248
-
list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) {
249
-
if (drm_mm_reserve_node(mm, &rsv->mm_node))
248
+
list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
249
+
if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
250
+
rsv->size, mm->chunk_size, &rsv->allocated,
251
+
DRM_BUDDY_RANGE_ALLOCATION))
252
+
continue;
253
+
254
+
block = amdgpu_vram_mgr_first_block(&rsv->allocated);
255
+
if (!block)
250
256
continue;
251
257
252
258
dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
253
-
rsv->mm_node.start, rsv->mm_node.size);
259
+
rsv->start, rsv->size);
254
260
255
-
vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
261
+
vis_usage = amdgpu_vram_mgr_vis_size(adev, block);
256
262
atomic64_add(vis_usage, &mgr->vis_usage);
257
263
spin_lock(&man->bdev->lru_lock);
258
-
man->usage += rsv->mm_node.size << PAGE_SHIFT;
264
+
man->usage += rsv->size;
259
265
spin_unlock(&man->bdev->lru_lock);
260
-
list_move(&rsv->node, &mgr->reserved_pages);
266
+
list_move(&rsv->blocks, &mgr->reserved_pages);
261
267
}
262
268
}
263
269
···
286
278
if (!rsv)
287
279
return -ENOMEM;
288
280
289
-
INIT_LIST_HEAD(&rsv->node);
290
-
rsv->mm_node.start = start >> PAGE_SHIFT;
291
-
rsv->mm_node.size = size >> PAGE_SHIFT;
281
+
INIT_LIST_HEAD(&rsv->allocated);
282
+
INIT_LIST_HEAD(&rsv->blocks);
292
283
293
-
spin_lock(&mgr->lock);
294
-
list_add_tail(&rsv->node, &mgr->reservations_pending);
284
+
rsv->start = start;
285
+
rsv->size = size;
286
+
287
+
mutex_lock(&mgr->lock);
288
+
list_add_tail(&rsv->blocks, &mgr->reservations_pending);
295
289
amdgpu_vram_mgr_do_reserve(&mgr->manager);
296
-
spin_unlock(&mgr->lock);
290
+
mutex_unlock(&mgr->lock);
297
291
298
292
return 0;
299
293
}
···
317
307
struct amdgpu_vram_reservation *rsv;
318
308
int ret;
319
309
320
-
spin_lock(&mgr->lock);
310
+
mutex_lock(&mgr->lock);
321
311
322
-
list_for_each_entry(rsv, &mgr->reservations_pending, node) {
323
-
if ((rsv->mm_node.start <= start) &&
324
-
(start < (rsv->mm_node.start + rsv->mm_node.size))) {
312
+
list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {
313
+
if (rsv->start <= start &&
314
+
(start < (rsv->start + rsv->size))) {
325
315
ret = -EBUSY;
326
316
goto out;
327
317
}
328
318
}
329
319
330
-
list_for_each_entry(rsv, &mgr->reserved_pages, node) {
331
-
if ((rsv->mm_node.start <= start) &&
332
-
(start < (rsv->mm_node.start + rsv->mm_node.size))) {
320
+
list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {
321
+
if (rsv->start <= start &&
322
+
(start < (rsv->start + rsv->size))) {
333
323
ret = 0;
334
324
goto out;
335
325
}
···
337
327
338
328
ret = -ENOENT;
339
329
out:
340
-
spin_unlock(&mgr->lock);
330
+
mutex_unlock(&mgr->lock);
341
331
return ret;
342
-
}
343
-
344
-
/**
345
-
* amdgpu_vram_mgr_virt_start - update virtual start address
346
-
*
347
-
* @mem: ttm_resource to update
348
-
* @node: just allocated node
349
-
*
350
-
* Calculate a virtual BO start address to easily check if everything is CPU
351
-
* accessible.
352
-
*/
353
-
static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem,
354
-
struct drm_mm_node *node)
355
-
{
356
-
unsigned long start;
357
-
358
-
start = node->start + node->size;
359
-
if (start > mem->num_pages)
360
-
start -= mem->num_pages;
361
-
else
362
-
start = 0;
363
-
mem->start = max(mem->start, start);
364
332
}
365
333
366
334
/**
···
356
368
const struct ttm_place *place,
357
369
struct ttm_resource **res)
358
370
{
359
-
unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages;
371
+
u64 vis_usage = 0, max_bytes, cur_size, min_block_size;
360
372
struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
361
373
struct amdgpu_device *adev = to_amdgpu_device(mgr);
362
-
uint64_t vis_usage = 0, mem_bytes, max_bytes;
363
-
struct ttm_range_mgr_node *node;
364
-
struct drm_mm *mm = &mgr->mm;
365
-
enum drm_mm_insert_mode mode;
366
-
unsigned i;
374
+
struct amdgpu_vram_mgr_resource *vres;
375
+
u64 size, remaining_size, lpfn, fpfn;
376
+
struct drm_buddy *mm = &mgr->mm;
377
+
struct drm_buddy_block *block;
378
+
unsigned long pages_per_block;
367
379
int r;
368
380
369
-
lpfn = place->lpfn;
381
+
lpfn = place->lpfn << PAGE_SHIFT;
370
382
if (!lpfn)
371
-
lpfn = man->size >> PAGE_SHIFT;
383
+
lpfn = man->size;
384
+
385
+
fpfn = place->fpfn << PAGE_SHIFT;
372
386
373
387
max_bytes = adev->gmc.mc_vram_size;
374
388
if (tbo->type != ttm_bo_type_kernel)
375
389
max_bytes -= AMDGPU_VM_RESERVED_VRAM;
376
390
377
-
mem_bytes = tbo->base.size;
378
391
if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
379
-
pages_per_node = ~0ul;
380
-
num_nodes = 1;
392
+
pages_per_block = ~0ul;
381
393
} else {
382
394
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
383
-
pages_per_node = HPAGE_PMD_NR;
395
+
pages_per_block = HPAGE_PMD_NR;
384
396
#else
385
397
/* default to 2MB */
386
-
pages_per_node = 2UL << (20UL - PAGE_SHIFT);
398
+
pages_per_block = 2UL << (20UL - PAGE_SHIFT);
387
399
#endif
388
-
pages_per_node = max_t(uint32_t, pages_per_node,
389
-
tbo->page_alignment);
390
-
num_nodes = DIV_ROUND_UP_ULL(PFN_UP(mem_bytes), pages_per_node);
400
+
pages_per_block = max_t(uint32_t, pages_per_block,
401
+
tbo->page_alignment);
391
402
}
392
403
393
-
node = kvmalloc(struct_size(node, mm_nodes, num_nodes),
394
-
GFP_KERNEL | __GFP_ZERO);
395
-
if (!node)
404
+
vres = kzalloc(sizeof(*vres), GFP_KERNEL);
405
+
if (!vres)
396
406
return -ENOMEM;
397
407
398
-
ttm_resource_init(tbo, place, &node->base);
408
+
ttm_resource_init(tbo, place, &vres->base);
399
409
400
410
/* bail out quickly if there's likely not enough VRAM for this BO */
401
411
if (ttm_resource_manager_usage(man) > max_bytes) {
···
401
415
goto error_fini;
402
416
}
403
417
404
-
mode = DRM_MM_INSERT_BEST;
418
+
INIT_LIST_HEAD(&vres->blocks);
419
+
405
420
if (place->flags & TTM_PL_FLAG_TOPDOWN)
406
-
mode = DRM_MM_INSERT_HIGH;
421
+
vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
407
422
408
-
pages_left = node->base.num_pages;
423
+
if (fpfn || lpfn != man->size)
424
+
/* Allocate blocks in desired range */
425
+
vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
409
426
410
-
/* Limit maximum size to 2GB due to SG table limitations */
411
-
pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
427
+
remaining_size = vres->base.num_pages << PAGE_SHIFT;
412
428
413
-
i = 0;
414
-
spin_lock(&mgr->lock);
415
-
while (pages_left) {
416
-
uint32_t alignment = tbo->page_alignment;
429
+
mutex_lock(&mgr->lock);
430
+
while (remaining_size) {
431
+
if (tbo->page_alignment)
432
+
min_block_size = tbo->page_alignment << PAGE_SHIFT;
433
+
else
434
+
min_block_size = mgr->default_page_size;
417
435
418
-
if (pages >= pages_per_node)
419
-
alignment = pages_per_node;
436
+
BUG_ON(min_block_size < mm->chunk_size);
420
437
421
-
r = drm_mm_insert_node_in_range(mm, &node->mm_nodes[i], pages,
422
-
alignment, 0, place->fpfn,
423
-
lpfn, mode);
424
-
if (unlikely(r)) {
425
-
if (pages > pages_per_node) {
426
-
if (is_power_of_2(pages))
427
-
pages = pages / 2;
428
-
else
429
-
pages = rounddown_pow_of_two(pages);
430
-
continue;
438
+
/* Limit maximum size to 2GiB due to SG table limitations */
439
+
size = min(remaining_size, 2ULL << 30);
440
+
441
+
if (size >= pages_per_block << PAGE_SHIFT)
442
+
min_block_size = pages_per_block << PAGE_SHIFT;
443
+
444
+
cur_size = size;
445
+
446
+
if (fpfn + size != place->lpfn << PAGE_SHIFT) {
447
+
/*
448
+
* Except for actual range allocation, modify the size and
449
+
* min_block_size conforming to continuous flag enablement
450
+
*/
451
+
if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
452
+
size = roundup_pow_of_two(size);
453
+
min_block_size = size;
454
+
/*
455
+
* Modify the size value if size is not
456
+
* aligned with min_block_size
457
+
*/
458
+
} else if (!IS_ALIGNED(size, min_block_size)) {
459
+
size = round_up(size, min_block_size);
431
460
}
432
-
goto error_free;
433
461
}
434
462
435
-
vis_usage += amdgpu_vram_mgr_vis_size(adev, &node->mm_nodes[i]);
436
-
amdgpu_vram_mgr_virt_start(&node->base, &node->mm_nodes[i]);
437
-
pages_left -= pages;
438
-
++i;
463
+
r = drm_buddy_alloc_blocks(mm, fpfn,
464
+
lpfn,
465
+
size,
466
+
min_block_size,
467
+
&vres->blocks,
468
+
vres->flags);
469
+
if (unlikely(r))
470
+
goto error_free_blocks;
439
471
440
-
if (pages > pages_left)
441
-
pages = pages_left;
472
+
if (size > remaining_size)
473
+
remaining_size = 0;
474
+
else
475
+
remaining_size -= size;
442
476
}
443
-
spin_unlock(&mgr->lock);
477
+
mutex_unlock(&mgr->lock);
444
478
445
-
if (i == 1)
446
-
node->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
479
+
if (cur_size != size) {
480
+
struct drm_buddy_block *block;
481
+
struct list_head *trim_list;
482
+
u64 original_size;
483
+
LIST_HEAD(temp);
484
+
485
+
trim_list = &vres->blocks;
486
+
original_size = vres->base.num_pages << PAGE_SHIFT;
487
+
488
+
/*
489
+
* If size value is rounded up to min_block_size, trim the last
490
+
* block to the required size
491
+
*/
492
+
if (!list_is_singular(&vres->blocks)) {
493
+
block = list_last_entry(&vres->blocks, typeof(*block), link);
494
+
list_move_tail(&block->link, &temp);
495
+
trim_list = &temp;
496
+
/*
497
+
* Compute the original_size value by subtracting the
498
+
* last block size with (aligned size - original size)
499
+
*/
500
+
original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size);
501
+
}
502
+
503
+
mutex_lock(&mgr->lock);
504
+
drm_buddy_block_trim(mm,
505
+
original_size,
506
+
trim_list);
507
+
mutex_unlock(&mgr->lock);
508
+
509
+
if (!list_empty(&temp))
510
+
list_splice_tail(trim_list, &vres->blocks);
511
+
}
512
+
513
+
list_for_each_entry(block, &vres->blocks, link)
514
+
vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
515
+
516
+
block = amdgpu_vram_mgr_first_block(&vres->blocks);
517
+
if (!block) {
518
+
r = -EINVAL;
519
+
goto error_fini;
520
+
}
521
+
522
+
vres->base.start = amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
523
+
524
+
if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks))
525
+
vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
447
526
448
527
if (adev->gmc.xgmi.connected_to_cpu)
449
-
node->base.bus.caching = ttm_cached;
528
+
vres->base.bus.caching = ttm_cached;
450
529
else
451
-
node->base.bus.caching = ttm_write_combined;
530
+
vres->base.bus.caching = ttm_write_combined;
452
531
453
532
atomic64_add(vis_usage, &mgr->vis_usage);
454
-
*res = &node->base;
533
+
*res = &vres->base;
455
534
return 0;
456
535
457
-
error_free:
458
-
while (i--)
459
-
drm_mm_remove_node(&node->mm_nodes[i]);
460
-
spin_unlock(&mgr->lock);
536
+
error_free_blocks:
537
+
drm_buddy_free_list(mm, &vres->blocks);
538
+
mutex_unlock(&mgr->lock);
461
539
error_fini:
462
-
ttm_resource_fini(man, &node->base);
463
-
kvfree(node);
540
+
ttm_resource_fini(man, &vres->base);
541
+
kfree(vres);
464
542
465
543
return r;
466
544
}
···
540
490
static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
541
491
struct ttm_resource *res)
542
492
{
543
-
struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res);
493
+
struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
544
494
struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
545
495
struct amdgpu_device *adev = to_amdgpu_device(mgr);
496
+
struct drm_buddy *mm = &mgr->mm;
497
+
struct drm_buddy_block *block;
546
498
uint64_t vis_usage = 0;
547
-
unsigned i, pages;
548
499
549
-
spin_lock(&mgr->lock);
550
-
for (i = 0, pages = res->num_pages; pages;
551
-
pages -= node->mm_nodes[i].size, ++i) {
552
-
struct drm_mm_node *mm = &node->mm_nodes[i];
500
+
mutex_lock(&mgr->lock);
501
+
list_for_each_entry(block, &vres->blocks, link)
502
+
vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
553
503
554
-
drm_mm_remove_node(mm);
555
-
vis_usage += amdgpu_vram_mgr_vis_size(adev, mm);
556
-
}
557
504
amdgpu_vram_mgr_do_reserve(man);
558
-
spin_unlock(&mgr->lock);
505
+
506
+
drm_buddy_free_list(mm, &vres->blocks);
507
+
mutex_unlock(&mgr->lock);
559
508
560
509
atomic64_sub(vis_usage, &mgr->vis_usage);
561
510
562
511
ttm_resource_fini(man, res);
563
-
kvfree(node);
512
+
kfree(vres);
564
513
}
565
514
566
515
/**
···
591
542
if (!*sgt)
592
543
return -ENOMEM;
593
544
594
-
/* Determine the number of DRM_MM nodes to export */
545
+
/* Determine the number of DRM_BUDDY blocks to export */
595
546
amdgpu_res_first(res, offset, length, &cursor);
596
547
while (cursor.remaining) {
597
548
num_entries++;
···
607
558
sg->length = 0;
608
559
609
560
/*
610
-
* Walk down DRM_MM nodes to populate scatterlist nodes
611
-
* @note: Use iterator api to get first the DRM_MM node
561
+
* Walk down DRM_BUDDY blocks to populate scatterlist nodes
562
+
* @note: Use iterator api to get first the DRM_BUDDY block
612
563
* and the number of bytes from it. Access the following
613
-
* DRM_MM node(s) if more buffer needs to exported
564
+
* DRM_BUDDY block(s) if more buffer needs to exported
614
565
*/
615
566
amdgpu_res_first(res, offset, length, &cursor);
616
567
for_each_sgtable_sg((*sgt), sg, i) {
···
697
648
struct drm_printer *printer)
698
649
{
699
650
struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
651
+
struct drm_buddy *mm = &mgr->mm;
652
+
struct drm_buddy_block *block;
700
653
701
654
drm_printf(printer, " vis usage:%llu\n",
702
655
amdgpu_vram_mgr_vis_usage(mgr));
703
656
704
-
spin_lock(&mgr->lock);
705
-
drm_mm_print(&mgr->mm, printer);
706
-
spin_unlock(&mgr->lock);
657
+
mutex_lock(&mgr->lock);
658
+
drm_printf(printer, "default_page_size: %lluKiB\n",
659
+
mgr->default_page_size >> 10);
660
+
661
+
drm_buddy_print(mm, printer);
662
+
663
+
drm_printf(printer, "reserved:\n");
664
+
list_for_each_entry(block, &mgr->reserved_pages, link)
665
+
drm_buddy_block_print(mm, block, printer);
666
+
mutex_unlock(&mgr->lock);
707
667
}
708
668
709
669
static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
···
732
674
{
733
675
struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
734
676
struct ttm_resource_manager *man = &mgr->manager;
677
+
int err;
735
678
736
679
ttm_resource_manager_init(man, &adev->mman.bdev,
737
680
adev->gmc.real_vram_size);
738
681
739
682
man->func = &amdgpu_vram_mgr_func;
740
683
741
-
drm_mm_init(&mgr->mm, 0, man->size >> PAGE_SHIFT);
742
-
spin_lock_init(&mgr->lock);
684
+
err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
685
+
if (err)
686
+
return err;
687
+
688
+
mutex_init(&mgr->lock);
743
689
INIT_LIST_HEAD(&mgr->reservations_pending);
744
690
INIT_LIST_HEAD(&mgr->reserved_pages);
691
+
mgr->default_page_size = PAGE_SIZE;
745
692
746
693
ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
747
694
ttm_resource_manager_set_used(man, true);
···
774
711
if (ret)
775
712
return;
776
713
777
-
spin_lock(&mgr->lock);
778
-
list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
714
+
mutex_lock(&mgr->lock);
715
+
list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)
779
716
kfree(rsv);
780
717
781
-
list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
782
-
drm_mm_remove_node(&rsv->mm_node);
718
+
list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
719
+
drm_buddy_free_list(&mgr->mm, &rsv->blocks);
783
720
kfree(rsv);
784
721
}
785
-
drm_mm_takedown(&mgr->mm);
786
-
spin_unlock(&mgr->lock);
722
+
drm_buddy_fini(&mgr->mm);
723
+
mutex_unlock(&mgr->lock);
787
724
788
725
ttm_resource_manager_cleanup(man);
789
726
ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
+89
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
+89
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
···
1
+
/* SPDX-License-Identifier: MIT
2
+
* Copyright 2021 Advanced Micro Devices, Inc.
3
+
*
4
+
* Permission is hereby granted, free of charge, to any person obtaining a
5
+
* copy of this software and associated documentation files (the "Software"),
6
+
* to deal in the Software without restriction, including without limitation
7
+
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
+
* and/or sell copies of the Software, and to permit persons to whom the
9
+
* Software is furnished to do so, subject to the following conditions:
10
+
*
11
+
* The above copyright notice and this permission notice shall be included in
12
+
* all copies or substantial portions of the Software.
13
+
*
14
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17
+
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
+
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
+
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
+
* OTHER DEALINGS IN THE SOFTWARE.
21
+
*
22
+
*/
23
+
24
+
#ifndef __AMDGPU_VRAM_MGR_H__
25
+
#define __AMDGPU_VRAM_MGR_H__
26
+
27
+
#include <drm/drm_buddy.h>
28
+
29
+
struct amdgpu_vram_mgr {
30
+
struct ttm_resource_manager manager;
31
+
struct drm_buddy mm;
32
+
/* protects access to buffer objects */
33
+
struct mutex lock;
34
+
struct list_head reservations_pending;
35
+
struct list_head reserved_pages;
36
+
atomic64_t vis_usage;
37
+
u64 default_page_size;
38
+
};
39
+
40
+
struct amdgpu_vram_mgr_resource {
41
+
struct ttm_resource base;
42
+
struct list_head blocks;
43
+
unsigned long flags;
44
+
};
45
+
46
+
static inline u64 amdgpu_vram_mgr_block_start(struct drm_buddy_block *block)
47
+
{
48
+
return drm_buddy_block_offset(block);
49
+
}
50
+
51
+
static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block)
52
+
{
53
+
return PAGE_SIZE << drm_buddy_block_order(block);
54
+
}
55
+
56
+
static inline struct drm_buddy_block *
57
+
amdgpu_vram_mgr_first_block(struct list_head *list)
58
+
{
59
+
return list_first_entry_or_null(list, struct drm_buddy_block, link);
60
+
}
61
+
62
+
static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
63
+
{
64
+
struct drm_buddy_block *block;
65
+
u64 start, size;
66
+
67
+
block = amdgpu_vram_mgr_first_block(head);
68
+
if (!block)
69
+
return false;
70
+
71
+
while (head != block->link.next) {
72
+
start = amdgpu_vram_mgr_block_start(block);
73
+
size = amdgpu_vram_mgr_block_size(block);
74
+
75
+
block = list_entry(block->link.next, struct drm_buddy_block, link);
76
+
if (start + size != amdgpu_vram_mgr_block_start(block))
77
+
return false;
78
+
}
79
+
80
+
return true;
81
+
}
82
+
83
+
static inline struct amdgpu_vram_mgr_resource *
84
+
to_amdgpu_vram_mgr_resource(struct ttm_resource *res)
85
+
{
86
+
return container_of(res, struct amdgpu_vram_mgr_resource, base);
87
+
}
88
+
89
+
#endif