Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_cache.h>
8#include <drm/drm_device.h>
9#include <drm/drm_gem.h>
10#include <drm/drm_gem_shmem_helper.h>
11#include <drm/drm_print.h>
12#include <drm/gpu_scheduler.h>
13#include <linux/dma-buf.h>
14#include <linux/dma-direct.h>
15#include <linux/iosys-map.h>
16#include <linux/pagemap.h>
17#include <linux/vmalloc.h>
18
19#include "amdxdna_ctx.h"
20#include "amdxdna_gem.h"
21#include "amdxdna_pci_drv.h"
22#include "amdxdna_ubuf.h"
23
24MODULE_IMPORT_NS("DMA_BUF");
25
26static int
27amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
28{
29 struct amdxdna_client *client = abo->client;
30 struct amdxdna_dev *xdna = client->xdna;
31 struct amdxdna_mem *mem = &abo->mem;
32 struct amdxdna_gem_obj *heap;
33 u32 align;
34 int ret;
35
36 mutex_lock(&client->mm_lock);
37
38 heap = client->dev_heap;
39 if (!heap) {
40 ret = -EINVAL;
41 goto unlock_out;
42 }
43
44 if (amdxdna_gem_uva(heap) == AMDXDNA_INVALID_ADDR) {
45 XDNA_ERR(xdna, "Invalid dev heap userptr");
46 ret = -EINVAL;
47 goto unlock_out;
48 }
49
50 if (mem->size == 0 || mem->size > heap->mem.size) {
51 XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, limit 0x%lx",
52 mem->size, heap->mem.size);
53 ret = -EINVAL;
54 goto unlock_out;
55 }
56
57 align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift);
58 ret = drm_mm_insert_node_generic(&heap->mm, &abo->mm_node,
59 mem->size, align,
60 0, DRM_MM_INSERT_BEST);
61 if (ret) {
62 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
63 goto unlock_out;
64 }
65
66 client->heap_usage += mem->size;
67
68 drm_gem_object_get(to_gobj(heap));
69
70unlock_out:
71 mutex_unlock(&client->mm_lock);
72
73 return ret;
74}
75
76static void
77amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo)
78{
79 struct amdxdna_client *client = abo->client;
80 struct amdxdna_gem_obj *heap;
81
82 mutex_lock(&client->mm_lock);
83
84 drm_mm_remove_node(&abo->mm_node);
85 client->heap_usage -= abo->mem.size;
86 heap = client->dev_heap;
87 drm_gem_object_put(to_gobj(heap));
88
89 mutex_unlock(&client->mm_lock);
90}
91
92static struct amdxdna_gem_obj *
93amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
94{
95 struct amdxdna_gem_obj *abo;
96
97 abo = kzalloc_obj(*abo);
98 if (!abo)
99 return ERR_PTR(-ENOMEM);
100
101 abo->pinned = false;
102 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
103 mutex_init(&abo->lock);
104
105 abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
106 abo->mem.uva = AMDXDNA_INVALID_ADDR;
107 abo->mem.size = size;
108 abo->open_ref = 0;
109 abo->internal = false;
110 INIT_LIST_HEAD(&abo->mem.umap_list);
111
112 return abo;
113}
114
115static void
116amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo)
117{
118 mutex_destroy(&abo->lock);
119 kfree(abo);
120}
121
122/*
123 * Obtains a kernel virtual address on the BO (usually of small size).
124 * The mapping is established on the first call and stays valid until
125 * amdxdna_gem_vunmap() is called.
126 */
127void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo)
128{
129 struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL);
130 int ret;
131
132 if (abo->mem.kva)
133 return abo->mem.kva;
134
135 /* The first call to get the kva, taking slow path. */
136 guard(mutex)(&abo->lock);
137
138 if (!abo->mem.kva) {
139 ret = drm_gem_vmap(to_gobj(abo), &map);
140 if (ret)
141 XDNA_ERR(abo->client->xdna, "Vmap bo failed, ret %d", ret);
142 else
143 abo->mem.kva = map.vaddr;
144 }
145 return abo->mem.kva;
146}
147
148/*
149 * Free mapping established through amdxdna_gem_vmap()
150 */
151static void amdxdna_gem_vunmap(struct amdxdna_gem_obj *abo)
152{
153 guard(mutex)(&abo->lock);
154
155 if (abo->mem.kva) {
156 struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva);
157
158 drm_gem_vunmap(to_gobj(abo), &map);
159 abo->mem.kva = NULL;
160 }
161}
162
163/*
164 * Obtain the user virtual address for accessing the BO.
165 * It can be used for device to access the BO when PASID is enabled.
166 */
167u64 amdxdna_gem_uva(struct amdxdna_gem_obj *abo)
168{
169 if (abo->type == AMDXDNA_BO_DEV) {
170 struct amdxdna_gem_obj *heap = abo->client->dev_heap;
171 u64 off = amdxdna_dev_bo_offset(abo);
172
173 if (amdxdna_gem_uva(heap) != AMDXDNA_INVALID_ADDR)
174 return amdxdna_gem_uva(heap) + off;
175 return AMDXDNA_INVALID_ADDR;
176 }
177
178 return abo->mem.uva;
179}
180
181/*
182 * Obtain the address for device to access the BO.
183 */
184u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo)
185{
186 if (abo->type == AMDXDNA_BO_DEV_HEAP)
187 return abo->client->xdna->dev_info->dev_mem_base;
188 if (abo->type == AMDXDNA_BO_DEV)
189 return abo->mm_node.start;
190 return amdxdna_obj_dma_addr(abo);
191}
192
193static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
194 const struct mmu_notifier_range *range,
195 unsigned long cur_seq)
196{
197 struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier);
198 struct amdxdna_gem_obj *abo = mapp->abo;
199 struct amdxdna_dev *xdna;
200
201 xdna = to_xdna_dev(to_gobj(abo)->dev);
202 XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d",
203 mapp->vma->vm_start, mapp->vma->vm_end, abo->type);
204
205 if (!mmu_notifier_range_blockable(range))
206 return false;
207
208 down_write(&xdna->notifier_lock);
209 abo->mem.map_invalid = true;
210 mapp->invalid = true;
211 mmu_interval_set_seq(&mapp->notifier, cur_seq);
212 up_write(&xdna->notifier_lock);
213
214 xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
215
216 if (range->event == MMU_NOTIFY_UNMAP) {
217 down_write(&xdna->notifier_lock);
218 if (!mapp->unmapped) {
219 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
220 mapp->unmapped = true;
221 }
222 up_write(&xdna->notifier_lock);
223 }
224
225 return true;
226}
227
228static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = {
229 .invalidate = amdxdna_hmm_invalidate,
230};
231
232static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
233 struct vm_area_struct *vma)
234{
235 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
236 struct amdxdna_umap *mapp;
237
238 down_read(&xdna->notifier_lock);
239 list_for_each_entry(mapp, &abo->mem.umap_list, node) {
240 if (!vma || mapp->vma == vma) {
241 if (!mapp->unmapped) {
242 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
243 mapp->unmapped = true;
244 }
245 if (vma)
246 break;
247 }
248 }
249 up_read(&xdna->notifier_lock);
250}
251
252static void amdxdna_umap_release(struct kref *ref)
253{
254 struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
255 struct amdxdna_gem_obj *abo = mapp->abo;
256 struct vm_area_struct *vma = mapp->vma;
257 struct amdxdna_dev *xdna;
258
259 mmu_interval_notifier_remove(&mapp->notifier);
260 if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping)
261 mapping_clear_unevictable(vma->vm_file->f_mapping);
262
263 xdna = to_xdna_dev(to_gobj(mapp->abo)->dev);
264 down_write(&xdna->notifier_lock);
265 list_del(&mapp->node);
266 if (list_empty(&abo->mem.umap_list))
267 abo->mem.uva = AMDXDNA_INVALID_ADDR;
268 up_write(&xdna->notifier_lock);
269
270 kvfree(mapp->range.hmm_pfns);
271 kfree(mapp);
272}
273
274void amdxdna_umap_put(struct amdxdna_umap *mapp)
275{
276 kref_put(&mapp->refcnt, amdxdna_umap_release);
277}
278
279static void amdxdna_hmm_unreg_work(struct work_struct *work)
280{
281 struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap,
282 hmm_unreg_work);
283
284 amdxdna_umap_put(mapp);
285}
286
287static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
288 struct vm_area_struct *vma)
289{
290 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
291 unsigned long len = vma->vm_end - vma->vm_start;
292 unsigned long addr = vma->vm_start;
293 struct amdxdna_umap *mapp;
294 u32 nr_pages;
295 int ret;
296
297 if (!xdna->dev_info->ops->hmm_invalidate)
298 return 0;
299
300 mapp = kzalloc_obj(*mapp);
301 if (!mapp)
302 return -ENOMEM;
303
304 nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT;
305 mapp->range.hmm_pfns = kvzalloc_objs(*mapp->range.hmm_pfns, nr_pages);
306 if (!mapp->range.hmm_pfns) {
307 ret = -ENOMEM;
308 goto free_map;
309 }
310
311 ret = mmu_interval_notifier_insert_locked(&mapp->notifier,
312 current->mm,
313 addr,
314 len,
315 &amdxdna_hmm_ops);
316 if (ret) {
317 XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret);
318 goto free_pfns;
319 }
320
321 mapp->range.notifier = &mapp->notifier;
322 mapp->range.start = vma->vm_start;
323 mapp->range.end = vma->vm_end;
324 mapp->range.default_flags = HMM_PFN_REQ_FAULT;
325 mapp->vma = vma;
326 mapp->abo = abo;
327 kref_init(&mapp->refcnt);
328
329 INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work);
330 if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping)
331 mapping_set_unevictable(vma->vm_file->f_mapping);
332
333 down_write(&xdna->notifier_lock);
334 if (list_empty(&abo->mem.umap_list))
335 abo->mem.uva = addr;
336 list_add_tail(&mapp->node, &abo->mem.umap_list);
337 up_write(&xdna->notifier_lock);
338
339 return 0;
340
341free_pfns:
342 kvfree(mapp->range.hmm_pfns);
343free_map:
344 kfree(mapp);
345 return ret;
346}
347
348static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj)
349{
350 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
351 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
352
353 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, amdxdna_gem_dev_addr(abo));
354 if (abo->pinned)
355 amdxdna_gem_unpin(abo);
356
357 amdxdna_gem_vunmap(abo);
358 amdxdna_gem_heap_free(abo);
359 drm_gem_object_release(gobj);
360 amdxdna_gem_destroy_obj(abo);
361}
362
363static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
364 struct vm_area_struct *vma)
365{
366 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
367 unsigned long num_pages = vma_pages(vma);
368 unsigned long offset = 0;
369 int ret;
370
371 if (!is_import_bo(abo)) {
372 ret = drm_gem_shmem_mmap(&abo->base, vma);
373 if (ret) {
374 XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
375 return ret;
376 }
377
378 /* The buffer is based on memory pages. Fix the flag. */
379 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
380 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
381 &num_pages);
382 if (ret) {
383 XDNA_ERR(xdna, "Failed insert pages %d", ret);
384 vma->vm_ops->close(vma);
385 return ret;
386 }
387
388 return 0;
389 }
390
391 vma->vm_private_data = NULL;
392 vma->vm_ops = NULL;
393 ret = dma_buf_mmap(abo->dma_buf, vma, 0);
394 if (ret) {
395 XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret);
396 return ret;
397 }
398
399 do {
400 vm_fault_t fault_ret;
401
402 fault_ret = handle_mm_fault(vma, vma->vm_start + offset,
403 FAULT_FLAG_WRITE, NULL);
404 if (fault_ret & VM_FAULT_ERROR) {
405 vma->vm_ops->close(vma);
406 XDNA_ERR(xdna, "Fault in page failed");
407 return -EFAULT;
408 }
409
410 offset += PAGE_SIZE;
411 } while (--num_pages);
412
413 /* Drop the reference drm_gem_mmap_obj() acquired.*/
414 drm_gem_object_put(to_gobj(abo));
415
416 return 0;
417}
418
419static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
420 struct vm_area_struct *vma)
421{
422 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
423 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
424 int ret;
425
426 ret = amdxdna_hmm_register(abo, vma);
427 if (ret)
428 return ret;
429
430 ret = amdxdna_insert_pages(abo, vma);
431 if (ret) {
432 XDNA_ERR(xdna, "Failed insert pages, ret %d", ret);
433 goto hmm_unreg;
434 }
435
436 XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
437 drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
438 vma->vm_start, gobj->size);
439 return 0;
440
441hmm_unreg:
442 amdxdna_hmm_unregister(abo, vma);
443 return ret;
444}
445
446static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
447{
448 struct drm_gem_object *gobj = dma_buf->priv;
449 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
450 unsigned long num_pages = vma_pages(vma);
451 int ret;
452
453 vma->vm_ops = &drm_gem_shmem_vm_ops;
454 vma->vm_private_data = gobj;
455
456 drm_gem_object_get(gobj);
457 ret = drm_gem_shmem_mmap(&abo->base, vma);
458 if (ret)
459 goto put_obj;
460
461 /* The buffer is based on memory pages. Fix the flag. */
462 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
463 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
464 &num_pages);
465 if (ret)
466 goto close_vma;
467
468 return 0;
469
470close_vma:
471 vma->vm_ops->close(vma);
472put_obj:
473 drm_gem_object_put(gobj);
474 return ret;
475}
476
477static const struct dma_buf_ops amdxdna_dmabuf_ops = {
478 .attach = drm_gem_map_attach,
479 .detach = drm_gem_map_detach,
480 .map_dma_buf = drm_gem_map_dma_buf,
481 .unmap_dma_buf = drm_gem_unmap_dma_buf,
482 .release = drm_gem_dmabuf_release,
483 .mmap = amdxdna_gem_dmabuf_mmap,
484 .vmap = drm_gem_dmabuf_vmap,
485 .vunmap = drm_gem_dmabuf_vunmap,
486};
487
488static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags)
489{
490 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
491 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
492
493 if (abo->dma_buf) {
494 get_dma_buf(abo->dma_buf);
495 return abo->dma_buf;
496 }
497
498 exp_info.ops = &amdxdna_dmabuf_ops;
499 exp_info.size = gobj->size;
500 exp_info.flags = flags;
501 exp_info.priv = gobj;
502 exp_info.resv = gobj->resv;
503
504 return drm_gem_dmabuf_export(gobj->dev, &exp_info);
505}
506
507static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo)
508{
509 dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL);
510 dma_buf_detach(abo->dma_buf, abo->attach);
511 dma_buf_put(abo->dma_buf);
512 drm_gem_object_release(to_gobj(abo));
513 kfree(abo);
514}
515
516static inline bool
517amdxdna_gem_skip_bo_usage(struct amdxdna_gem_obj *abo)
518{
519 /* Do not count imported BOs since the buffer is not allocated by us. */
520 if (is_import_bo(abo))
521 return true;
522
523 /* Already counted as part of HEAP BO */
524 if (abo->type == AMDXDNA_BO_DEV)
525 return true;
526
527 return false;
528}
529
530static void
531amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo)
532{
533 struct amdxdna_client *client = abo->client;
534
535 if (amdxdna_gem_skip_bo_usage(abo))
536 return;
537
538 guard(mutex)(&client->mm_lock);
539
540 client->total_bo_usage += abo->mem.size;
541 if (abo->internal)
542 client->total_int_bo_usage += abo->mem.size;
543}
544
545static void
546amdxdna_gem_del_bo_usage(struct amdxdna_gem_obj *abo)
547{
548 struct amdxdna_client *client = abo->client;
549
550 if (amdxdna_gem_skip_bo_usage(abo))
551 return;
552
553 guard(mutex)(&client->mm_lock);
554
555 client->total_bo_usage -= abo->mem.size;
556 if (abo->internal)
557 client->total_int_bo_usage -= abo->mem.size;
558}
559
560static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
561{
562 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
563 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
564
565 amdxdna_hmm_unregister(abo, NULL);
566 flush_workqueue(xdna->notifier_wq);
567
568 if (abo->pinned)
569 amdxdna_gem_unpin(abo);
570
571 if (abo->type == AMDXDNA_BO_DEV_HEAP)
572 drm_mm_takedown(&abo->mm);
573
574 if (amdxdna_iova_on(xdna))
575 amdxdna_iommu_unmap_bo(xdna, abo);
576
577 amdxdna_gem_vunmap(abo);
578 mutex_destroy(&abo->lock);
579
580 if (is_import_bo(abo))
581 amdxdna_imported_obj_free(abo);
582 else
583 drm_gem_shmem_free(&abo->base);
584}
585
586static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *filp)
587{
588 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
589 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
590 int ret;
591
592 guard(mutex)(&abo->lock);
593 abo->open_ref++;
594
595 if (abo->open_ref == 1) {
596 /* Attached to the client when first opened by it. */
597 abo->client = filp->driver_priv;
598 amdxdna_gem_add_bo_usage(abo);
599 }
600 if (amdxdna_iova_on(xdna)) {
601 ret = amdxdna_iommu_map_bo(xdna, abo);
602 if (ret)
603 return ret;
604 }
605
606 return 0;
607}
608
609static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
610{
611 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
612
613 guard(mutex)(&abo->lock);
614 abo->open_ref--;
615
616 if (abo->open_ref == 0) {
617 amdxdna_gem_del_bo_usage(abo);
618 /* Detach from the client when last closed by it. */
619 abo->client = NULL;
620 }
621}
622
623static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
624{
625 struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
626 void *base = amdxdna_gem_vmap(abo->client->dev_heap);
627 u64 offset = amdxdna_dev_bo_offset(abo);
628
629 if (!base)
630 return -ENOMEM;
631 iosys_map_set_vaddr(map, base + offset);
632 return 0;
633}
634
635static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
636 .free = amdxdna_gem_dev_obj_free,
637 .vmap = amdxdna_gem_dev_obj_vmap,
638};
639
640static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
641 .free = amdxdna_gem_obj_free,
642 .open = amdxdna_gem_obj_open,
643 .close = amdxdna_gem_obj_close,
644 .print_info = drm_gem_shmem_object_print_info,
645 .pin = drm_gem_shmem_object_pin,
646 .unpin = drm_gem_shmem_object_unpin,
647 .get_sg_table = drm_gem_shmem_object_get_sg_table,
648 .vmap = drm_gem_shmem_object_vmap,
649 .vunmap = drm_gem_shmem_object_vunmap,
650 .mmap = amdxdna_gem_obj_mmap,
651 .vm_ops = &drm_gem_shmem_vm_ops,
652 .export = amdxdna_gem_prime_export,
653};
654
655/* For drm_driver->gem_create_object callback */
656struct drm_gem_object *
657amdxdna_gem_create_shmem_object_cb(struct drm_device *dev, size_t size)
658{
659 struct amdxdna_gem_obj *abo;
660
661 abo = amdxdna_gem_create_obj(dev, size);
662 if (IS_ERR(abo))
663 return ERR_CAST(abo);
664
665 to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs;
666
667 return to_gobj(abo);
668}
669
670static struct amdxdna_gem_obj *
671amdxdna_gem_create_shmem_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args)
672{
673 size_t size = args->size;
674 struct drm_gem_shmem_object *shmem = drm_gem_shmem_create(dev, size);
675
676 if (IS_ERR(shmem))
677 return ERR_CAST(shmem);
678
679 shmem->map_wc = false;
680 return to_xdna_obj(&shmem->base);
681}
682
683static struct amdxdna_gem_obj *
684amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args)
685{
686 struct amdxdna_dev *xdna = to_xdna_dev(dev);
687 struct amdxdna_drm_va_tbl va_tbl;
688 struct drm_gem_object *gobj;
689 struct dma_buf *dma_buf;
690
691 if (copy_from_user(&va_tbl, u64_to_user_ptr(args->vaddr), sizeof(va_tbl))) {
692 XDNA_DBG(xdna, "Access va table failed");
693 return ERR_PTR(-EINVAL);
694 }
695
696 if (va_tbl.num_entries) {
697 dma_buf = amdxdna_get_ubuf(dev, va_tbl.num_entries,
698 u64_to_user_ptr(args->vaddr + sizeof(va_tbl)));
699 } else {
700 dma_buf = dma_buf_get(va_tbl.dmabuf_fd);
701 }
702
703 if (IS_ERR(dma_buf))
704 return ERR_CAST(dma_buf);
705
706 gobj = amdxdna_gem_prime_import(dev, dma_buf);
707 if (IS_ERR(gobj)) {
708 dma_buf_put(dma_buf);
709 return ERR_CAST(gobj);
710 }
711
712 dma_buf_put(dma_buf);
713
714 return to_xdna_obj(gobj);
715}
716
717struct drm_gem_object *
718amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
719{
720 struct dma_buf_attachment *attach;
721 struct amdxdna_gem_obj *abo;
722 struct drm_gem_object *gobj;
723 struct sg_table *sgt;
724 int ret;
725
726 get_dma_buf(dma_buf);
727
728 attach = dma_buf_attach(dma_buf, dev->dev);
729 if (IS_ERR(attach)) {
730 ret = PTR_ERR(attach);
731 goto put_buf;
732 }
733
734 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
735 if (IS_ERR(sgt)) {
736 ret = PTR_ERR(sgt);
737 goto fail_detach;
738 }
739
740 gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
741 if (IS_ERR(gobj)) {
742 ret = PTR_ERR(gobj);
743 goto fail_unmap;
744 }
745
746 abo = to_xdna_obj(gobj);
747 abo->attach = attach;
748 abo->dma_buf = dma_buf;
749 abo->type = AMDXDNA_BO_SHARE;
750 gobj->resv = dma_buf->resv;
751
752 return gobj;
753
754fail_unmap:
755 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
756fail_detach:
757 dma_buf_detach(dma_buf, attach);
758put_buf:
759 dma_buf_put(dma_buf);
760
761 return ERR_PTR(ret);
762}
763
764static struct amdxdna_gem_obj *
765amdxdna_drm_create_share_bo(struct drm_device *dev,
766 struct amdxdna_drm_create_bo *args, struct drm_file *filp)
767{
768 struct amdxdna_gem_obj *abo;
769
770 if (args->vaddr)
771 abo = amdxdna_gem_create_ubuf_object(dev, args);
772 else
773 abo = amdxdna_gem_create_shmem_object(dev, args);
774 if (IS_ERR(abo))
775 return ERR_CAST(abo);
776
777 if (args->type == AMDXDNA_BO_DEV_HEAP) {
778 abo->type = AMDXDNA_BO_DEV_HEAP;
779 abo->internal = true;
780 } else {
781 abo->type = AMDXDNA_BO_SHARE;
782 abo->internal = args->type == AMDXDNA_BO_CMD;
783 }
784
785 return abo;
786}
787
788static struct amdxdna_gem_obj *
789amdxdna_drm_create_dev_heap_bo(struct drm_device *dev,
790 struct amdxdna_drm_create_bo *args, struct drm_file *filp)
791{
792 struct amdxdna_client *client = filp->driver_priv;
793 struct amdxdna_dev *xdna = to_xdna_dev(dev);
794 struct amdxdna_gem_obj *abo;
795 int ret;
796
797 WARN_ON(!is_power_of_2(xdna->dev_info->dev_mem_size));
798 XDNA_DBG(xdna, "Requested dev heap size 0x%llx", args->size);
799 if (!args->size || !IS_ALIGNED(args->size, xdna->dev_info->dev_mem_size)) {
800 XDNA_ERR(xdna, "The dev heap size 0x%llx is not multiple of 0x%lx",
801 args->size, xdna->dev_info->dev_mem_size);
802 return ERR_PTR(-EINVAL);
803 }
804
805 /* HEAP BO is a special case of SHARE BO. */
806 abo = amdxdna_drm_create_share_bo(dev, args, filp);
807 if (IS_ERR(abo))
808 return ERR_CAST(abo);
809
810 /* Set up heap for this client. */
811 mutex_lock(&client->mm_lock);
812
813 if (client->dev_heap) {
814 XDNA_DBG(client->xdna, "dev heap is already created");
815 ret = -EBUSY;
816 goto mm_unlock;
817 }
818 client->dev_heap = abo;
819 drm_gem_object_get(to_gobj(abo));
820
821 drm_mm_init(&abo->mm, xdna->dev_info->dev_mem_base, abo->mem.size);
822
823 mutex_unlock(&client->mm_lock);
824
825 return abo;
826
827mm_unlock:
828 mutex_unlock(&client->mm_lock);
829 drm_gem_object_put(to_gobj(abo));
830 return ERR_PTR(ret);
831}
832
833struct amdxdna_gem_obj *
834amdxdna_drm_create_dev_bo(struct drm_device *dev,
835 struct amdxdna_drm_create_bo *args, struct drm_file *filp)
836{
837 size_t aligned_sz = PAGE_ALIGN(args->size);
838 struct amdxdna_client *client = filp->driver_priv;
839 struct amdxdna_dev *xdna = to_xdna_dev(dev);
840 struct amdxdna_gem_obj *abo;
841 struct drm_gem_object *gobj;
842 int ret;
843
844 if (!aligned_sz) {
845 XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size);
846 return ERR_PTR(-EINVAL);
847 }
848
849 abo = amdxdna_gem_create_obj(dev, aligned_sz);
850 if (IS_ERR(abo))
851 return abo;
852 gobj = to_gobj(abo);
853 gobj->funcs = &amdxdna_gem_dev_obj_funcs;
854 abo->type = AMDXDNA_BO_DEV;
855 abo->internal = true;
856 /*
857 * DEV BOs cannot be alive when client is gone, it's OK to
858 * always establish the connection.
859 */
860 abo->client = client;
861
862 ret = amdxdna_gem_heap_alloc(abo);
863 if (ret) {
864 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
865 amdxdna_gem_destroy_obj(abo);
866 return ERR_PTR(ret);
867 }
868 drm_gem_private_object_init(dev, gobj, aligned_sz);
869
870 return abo;
871}
872
873int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
874{
875 struct amdxdna_dev *xdna = to_xdna_dev(dev);
876 struct amdxdna_drm_create_bo *args = data;
877 struct amdxdna_gem_obj *abo;
878 int ret;
879
880 if (args->flags)
881 return -EINVAL;
882
883 XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx",
884 args->type, args->vaddr, args->size, args->flags);
885 switch (args->type) {
886 case AMDXDNA_BO_CMD:
887 fallthrough;
888 case AMDXDNA_BO_SHARE:
889 abo = amdxdna_drm_create_share_bo(dev, args, filp);
890 break;
891 case AMDXDNA_BO_DEV_HEAP:
892 abo = amdxdna_drm_create_dev_heap_bo(dev, args, filp);
893 break;
894 case AMDXDNA_BO_DEV:
895 abo = amdxdna_drm_create_dev_bo(dev, args, filp);
896 break;
897 default:
898 return -EINVAL;
899 }
900 if (IS_ERR(abo))
901 return PTR_ERR(abo);
902
903 /* Ready to publish object to userspace and count for BO usage. */
904 ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle);
905 if (ret) {
906 XDNA_ERR(xdna, "Create handle failed");
907 goto put_obj;
908 }
909
910 XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx",
911 args->handle, args->type, amdxdna_gem_uva(abo),
912 amdxdna_gem_dev_addr(abo), abo->mem.size);
913put_obj:
914 /* Dereference object reference. Handle holds it now. */
915 drm_gem_object_put(to_gobj(abo));
916 return ret;
917}
918
919int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
920{
921 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
922 int ret;
923
924 if (abo->type == AMDXDNA_BO_DEV)
925 abo = abo->client->dev_heap;
926
927 if (is_import_bo(abo))
928 return 0;
929
930 ret = drm_gem_shmem_pin(&abo->base);
931
932 XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret);
933 return ret;
934}
935
936int amdxdna_gem_pin(struct amdxdna_gem_obj *abo)
937{
938 int ret;
939
940 mutex_lock(&abo->lock);
941 ret = amdxdna_gem_pin_nolock(abo);
942 mutex_unlock(&abo->lock);
943
944 return ret;
945}
946
947void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
948{
949 if (abo->type == AMDXDNA_BO_DEV)
950 abo = abo->client->dev_heap;
951
952 if (is_import_bo(abo))
953 return;
954
955 mutex_lock(&abo->lock);
956 drm_gem_shmem_unpin(&abo->base);
957 mutex_unlock(&abo->lock);
958}
959
960struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client,
961 u32 bo_hdl, u8 bo_type)
962{
963 struct amdxdna_gem_obj *abo;
964 struct drm_gem_object *gobj;
965
966 gobj = drm_gem_object_lookup(client->filp, bo_hdl);
967 if (!gobj) {
968 XDNA_DBG(client->xdna, "Can not find bo %d", bo_hdl);
969 return NULL;
970 }
971
972 abo = to_xdna_obj(gobj);
973 if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type)
974 return abo;
975
976 drm_gem_object_put(gobj);
977 return NULL;
978}
979
980int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
981{
982 struct amdxdna_drm_get_bo_info *args = data;
983 struct amdxdna_dev *xdna = to_xdna_dev(dev);
984 struct amdxdna_gem_obj *abo;
985 struct drm_gem_object *gobj;
986 int ret = 0;
987
988 if (args->ext || args->ext_flags || args->pad)
989 return -EINVAL;
990
991 gobj = drm_gem_object_lookup(filp, args->handle);
992 if (!gobj) {
993 XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle);
994 return -ENOENT;
995 }
996
997 abo = to_xdna_obj(gobj);
998 args->vaddr = amdxdna_gem_uva(abo);
999 args->xdna_addr = amdxdna_gem_dev_addr(abo);
1000
1001 if (abo->type != AMDXDNA_BO_DEV)
1002 args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node);
1003 else
1004 args->map_offset = AMDXDNA_INVALID_ADDR;
1005
1006 XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx",
1007 args->handle, args->map_offset, args->vaddr, args->xdna_addr);
1008
1009 drm_gem_object_put(gobj);
1010 return ret;
1011}
1012
1013/*
1014 * The sync bo ioctl is to make sure the CPU cache is in sync with memory.
1015 * This is required because NPU is not cache coherent device. CPU cache
1016 * flushing/invalidation is expensive so it is best to handle this outside
1017 * of the command submission path. This ioctl allows explicit cache
1018 * flushing/invalidation outside of the critical path.
1019 */
1020int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
1021 void *data, struct drm_file *filp)
1022{
1023 struct amdxdna_dev *xdna = to_xdna_dev(dev);
1024 struct amdxdna_drm_sync_bo *args = data;
1025 struct amdxdna_gem_obj *abo;
1026 struct drm_gem_object *gobj;
1027 int ret;
1028
1029 gobj = drm_gem_object_lookup(filp, args->handle);
1030 if (!gobj) {
1031 XDNA_ERR(xdna, "Lookup GEM object failed");
1032 return -ENOENT;
1033 }
1034 abo = to_xdna_obj(gobj);
1035
1036 ret = amdxdna_gem_pin(abo);
1037 if (ret) {
1038 XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret);
1039 goto put_obj;
1040 }
1041
1042 if (is_import_bo(abo))
1043 drm_clflush_sg(abo->base.sgt);
1044 else if (amdxdna_gem_vmap(abo))
1045 drm_clflush_virt_range(amdxdna_gem_vmap(abo) + args->offset, args->size);
1046 else if (abo->base.pages)
1047 drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT);
1048 else
1049 drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
1050
1051 amdxdna_gem_unpin(abo);
1052
1053 XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
1054 args->handle, args->offset, args->size);
1055
1056 if (args->direction == SYNC_DIRECT_FROM_DEVICE)
1057 ret = amdxdna_hwctx_sync_debug_bo(abo->client, args->handle);
1058
1059put_obj:
1060 drm_gem_object_put(gobj);
1061 return ret;
1062}
1063
1064int amdxdna_drm_get_bo_usage(struct drm_device *dev, struct amdxdna_drm_get_array *args)
1065{
1066 size_t min_sz = min(args->element_size, sizeof(struct amdxdna_drm_bo_usage));
1067 char __user *buf = u64_to_user_ptr(args->buffer);
1068 struct amdxdna_dev *xdna = to_xdna_dev(dev);
1069 struct amdxdna_client *tmp_client;
1070 struct amdxdna_drm_bo_usage tmp;
1071
1072 drm_WARN_ON(dev, !mutex_is_locked(&xdna->dev_lock));
1073
1074 if (args->num_element != 1)
1075 return -EINVAL;
1076
1077 if (copy_from_user(&tmp, buf, min_sz))
1078 return -EFAULT;
1079
1080 if (!tmp.pid)
1081 return -EINVAL;
1082
1083 tmp.total_usage = 0;
1084 tmp.internal_usage = 0;
1085 tmp.heap_usage = 0;
1086
1087 list_for_each_entry(tmp_client, &xdna->client_list, node) {
1088 if (tmp.pid != tmp_client->pid)
1089 continue;
1090
1091 mutex_lock(&tmp_client->mm_lock);
1092 tmp.total_usage += tmp_client->total_bo_usage;
1093 tmp.internal_usage += tmp_client->total_int_bo_usage;
1094 tmp.heap_usage += tmp_client->heap_usage;
1095 mutex_unlock(&tmp_client->mm_lock);
1096 }
1097
1098 if (copy_to_user(buf, &tmp, min_sz))
1099 return -EFAULT;
1100
1101 return 0;
1102}