Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
accel/ivpu: Add support for userptr buffer objects
Introduce a new ioctl `drm_ivpu_bo_create_from_userptr` that allows
users to create GEM buffer objects from user pointers to memory regions.
The user pointer must be page-aligned and the memory region must remain
valid for the buffer object's lifetime.
Userptr buffers enable direct use of mmapped files (e.g. inference
weights) in NPU workloads without copying data to NPU buffer objects.
This reduces memory usage and provides better flexibility for NPU
applications.
Signed-off-by: Jacek Lawrynowicz <jacek.lawrynowicz@linux.intel.com>
Reviewed-by: Jeff Hugo <jeff.hugo@oss.qualcomm.com>
Signed-off-by: Karol Wachowski <karol.wachowski@linux.intel.com>
Link: https://patch.msgid.link/20251029091752.203198-1-karol.wachowski@linux.intel.com
authored by
Jacek Lawrynowicz
and committed by
Karol Wachowski
57557964
00812636
+270
-3
+1
drivers/accel/ivpu/Makefile
+1
drivers/accel/ivpu/Makefile
+3
drivers/accel/ivpu/ivpu_drv.c
+3
drivers/accel/ivpu/ivpu_drv.c
···
134
134
return true;
135
135
case DRM_IVPU_CAP_DMA_MEMORY_RANGE:
136
136
return true;
137
+
case DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR:
138
+
return true;
137
139
case DRM_IVPU_CAP_MANAGE_CMDQ:
138
140
return vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW;
139
141
default:
···
315
313
DRM_IOCTL_DEF_DRV(IVPU_CMDQ_CREATE, ivpu_cmdq_create_ioctl, 0),
316
314
DRM_IOCTL_DEF_DRV(IVPU_CMDQ_DESTROY, ivpu_cmdq_destroy_ioctl, 0),
317
315
DRM_IOCTL_DEF_DRV(IVPU_CMDQ_SUBMIT, ivpu_cmdq_submit_ioctl, 0),
316
+
DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE_FROM_USERPTR, ivpu_bo_create_from_userptr_ioctl, 0),
318
317
};
319
318
320
319
static int ivpu_wait_for_ready(struct ivpu_device *vdev)
+1
-1
drivers/accel/ivpu/ivpu_gem.c
+1
-1
drivers/accel/ivpu/ivpu_gem.c
···
96
96
if (!bo->mmu_mapped) {
97
97
drm_WARN_ON(&vdev->drm, !bo->ctx);
98
98
ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
99
-
ivpu_bo_is_snooped(bo));
99
+
ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo));
100
100
if (ret) {
101
101
ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
102
102
goto unlock;
+7
drivers/accel/ivpu/ivpu_gem.h
+7
drivers/accel/ivpu/ivpu_gem.h
···
38
38
int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
39
39
int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
40
40
int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
41
+
int ivpu_bo_create_from_userptr_ioctl(struct drm_device *dev, void *data,
42
+
struct drm_file *file);
41
43
42
44
void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
43
45
void ivpu_bo_list_print(struct drm_device *dev);
···
75
73
return true;
76
74
77
75
return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED;
76
+
}
77
+
78
+
static inline bool ivpu_bo_is_read_only(struct ivpu_bo *bo)
79
+
{
80
+
return bo->flags & DRM_IVPU_BO_READ_ONLY;
78
81
}
79
82
80
83
static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr)
+202
drivers/accel/ivpu/ivpu_gem_userptr.c
+202
drivers/accel/ivpu/ivpu_gem_userptr.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2020-2025 Intel Corporation
4
+
*/
5
+
6
+
#include <linux/dma-buf.h>
7
+
#include <linux/err.h>
8
+
#include <linux/highmem.h>
9
+
#include <linux/mm.h>
10
+
#include <linux/mman.h>
11
+
#include <linux/scatterlist.h>
12
+
#include <linux/slab.h>
13
+
#include <linux/capability.h>
14
+
15
+
#include <drm/drm_device.h>
16
+
#include <drm/drm_file.h>
17
+
#include <drm/drm_gem.h>
18
+
19
+
#include "ivpu_drv.h"
20
+
#include "ivpu_gem.h"
21
+
22
+
static struct sg_table *
23
+
ivpu_gem_userptr_dmabuf_map(struct dma_buf_attachment *attachment,
24
+
enum dma_data_direction direction)
25
+
{
26
+
struct sg_table *sgt = attachment->dmabuf->priv;
27
+
int ret;
28
+
29
+
ret = dma_map_sgtable(attachment->dev, sgt, direction, DMA_ATTR_SKIP_CPU_SYNC);
30
+
if (ret)
31
+
return ERR_PTR(ret);
32
+
33
+
return sgt;
34
+
}
35
+
36
+
static void ivpu_gem_userptr_dmabuf_unmap(struct dma_buf_attachment *attachment,
37
+
struct sg_table *sgt,
38
+
enum dma_data_direction direction)
39
+
{
40
+
dma_unmap_sgtable(attachment->dev, sgt, direction, DMA_ATTR_SKIP_CPU_SYNC);
41
+
}
42
+
43
+
static void ivpu_gem_userptr_dmabuf_release(struct dma_buf *dma_buf)
44
+
{
45
+
struct sg_table *sgt = dma_buf->priv;
46
+
struct sg_page_iter page_iter;
47
+
struct page *page;
48
+
49
+
for_each_sgtable_page(sgt, &page_iter, 0) {
50
+
page = sg_page_iter_page(&page_iter);
51
+
unpin_user_page(page);
52
+
}
53
+
54
+
sg_free_table(sgt);
55
+
kfree(sgt);
56
+
}
57
+
58
+
static const struct dma_buf_ops ivpu_gem_userptr_dmabuf_ops = {
59
+
.map_dma_buf = ivpu_gem_userptr_dmabuf_map,
60
+
.unmap_dma_buf = ivpu_gem_userptr_dmabuf_unmap,
61
+
.release = ivpu_gem_userptr_dmabuf_release,
62
+
};
63
+
64
+
static struct dma_buf *
65
+
ivpu_create_userptr_dmabuf(struct ivpu_device *vdev, void __user *user_ptr,
66
+
size_t size, uint32_t flags)
67
+
{
68
+
struct dma_buf_export_info exp_info = {};
69
+
struct dma_buf *dma_buf;
70
+
struct sg_table *sgt;
71
+
struct page **pages;
72
+
unsigned long nr_pages = size >> PAGE_SHIFT;
73
+
unsigned int gup_flags = FOLL_LONGTERM;
74
+
int ret, i, pinned;
75
+
76
+
/* Add FOLL_WRITE only if the BO is not read-only */
77
+
if (!(flags & DRM_IVPU_BO_READ_ONLY))
78
+
gup_flags |= FOLL_WRITE;
79
+
80
+
pages = kvmalloc_array(nr_pages, sizeof(*pages), GFP_KERNEL);
81
+
if (!pages)
82
+
return ERR_PTR(-ENOMEM);
83
+
84
+
pinned = pin_user_pages_fast((unsigned long)user_ptr, nr_pages, gup_flags, pages);
85
+
if (pinned < 0) {
86
+
ret = pinned;
87
+
ivpu_warn(vdev, "Failed to pin user pages: %d\n", ret);
88
+
goto free_pages_array;
89
+
}
90
+
91
+
if (pinned != nr_pages) {
92
+
ivpu_warn(vdev, "Pinned %d pages, expected %lu\n", pinned, nr_pages);
93
+
ret = -EFAULT;
94
+
goto unpin_pages;
95
+
}
96
+
97
+
sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
98
+
if (!sgt) {
99
+
ret = -ENOMEM;
100
+
goto unpin_pages;
101
+
}
102
+
103
+
ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, 0, size, GFP_KERNEL);
104
+
if (ret) {
105
+
ivpu_warn(vdev, "Failed to create sg table: %d\n", ret);
106
+
goto free_sgt;
107
+
}
108
+
109
+
exp_info.exp_name = "ivpu_userptr_dmabuf";
110
+
exp_info.owner = THIS_MODULE;
111
+
exp_info.ops = &ivpu_gem_userptr_dmabuf_ops;
112
+
exp_info.size = size;
113
+
exp_info.flags = O_RDWR | O_CLOEXEC;
114
+
exp_info.priv = sgt;
115
+
116
+
dma_buf = dma_buf_export(&exp_info);
117
+
if (IS_ERR(dma_buf)) {
118
+
ret = PTR_ERR(dma_buf);
119
+
ivpu_warn(vdev, "Failed to export userptr dma-buf: %d\n", ret);
120
+
goto free_sg_table;
121
+
}
122
+
123
+
kvfree(pages);
124
+
return dma_buf;
125
+
126
+
free_sg_table:
127
+
sg_free_table(sgt);
128
+
free_sgt:
129
+
kfree(sgt);
130
+
unpin_pages:
131
+
for (i = 0; i < pinned; i++)
132
+
unpin_user_page(pages[i]);
133
+
free_pages_array:
134
+
kvfree(pages);
135
+
return ERR_PTR(ret);
136
+
}
137
+
138
+
static struct ivpu_bo *
139
+
ivpu_bo_create_from_userptr(struct ivpu_device *vdev, void __user *user_ptr,
140
+
size_t size, uint32_t flags)
141
+
{
142
+
struct dma_buf *dma_buf;
143
+
struct drm_gem_object *obj;
144
+
struct ivpu_bo *bo;
145
+
146
+
dma_buf = ivpu_create_userptr_dmabuf(vdev, user_ptr, size, flags);
147
+
if (IS_ERR(dma_buf))
148
+
return ERR_CAST(dma_buf);
149
+
150
+
obj = ivpu_gem_prime_import(&vdev->drm, dma_buf);
151
+
if (IS_ERR(obj)) {
152
+
dma_buf_put(dma_buf);
153
+
return ERR_CAST(obj);
154
+
}
155
+
156
+
dma_buf_put(dma_buf);
157
+
158
+
bo = to_ivpu_bo(obj);
159
+
bo->flags = flags;
160
+
161
+
return bo;
162
+
}
163
+
164
+
int ivpu_bo_create_from_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
165
+
{
166
+
struct drm_ivpu_bo_create_from_userptr *args = data;
167
+
struct ivpu_file_priv *file_priv = file->driver_priv;
168
+
struct ivpu_device *vdev = to_ivpu_device(dev);
169
+
void __user *user_ptr = u64_to_user_ptr(args->user_ptr);
170
+
struct ivpu_bo *bo;
171
+
int ret;
172
+
173
+
if (args->flags & ~(DRM_IVPU_BO_HIGH_MEM | DRM_IVPU_BO_DMA_MEM | DRM_IVPU_BO_READ_ONLY))
174
+
return -EINVAL;
175
+
176
+
if (!args->user_ptr || !args->size)
177
+
return -EINVAL;
178
+
179
+
if (!PAGE_ALIGNED(args->user_ptr) || !PAGE_ALIGNED(args->size))
180
+
return -EINVAL;
181
+
182
+
if (!access_ok(user_ptr, args->size))
183
+
return -EFAULT;
184
+
185
+
bo = ivpu_bo_create_from_userptr(vdev, user_ptr, args->size, args->flags);
186
+
if (IS_ERR(bo))
187
+
return PTR_ERR(bo);
188
+
189
+
ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
190
+
if (ret) {
191
+
ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
192
+
bo, file_priv->ctx.id, args->size, args->flags);
193
+
} else {
194
+
ivpu_dbg(vdev, BO, "Created userptr BO: handle=%u vpu_addr=0x%llx size=%llu flags=0x%x\n",
195
+
args->handle, bo->vpu_addr, args->size, bo->flags);
196
+
args->vpu_addr = bo->vpu_addr;
197
+
}
198
+
199
+
drm_gem_object_put(&bo->base.base);
200
+
201
+
return ret;
202
+
}
+3
-1
drivers/accel/ivpu/ivpu_mmu_context.c
+3
-1
drivers/accel/ivpu/ivpu_mmu_context.c
···
430
430
431
431
int
432
432
ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
433
-
u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)
433
+
u64 vpu_addr, struct sg_table *sgt, bool llc_coherent, bool read_only)
434
434
{
435
435
size_t start_vpu_addr = vpu_addr;
436
436
struct scatterlist *sg;
···
450
450
prot = IVPU_MMU_ENTRY_MAPPED;
451
451
if (llc_coherent)
452
452
prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
453
+
if (read_only)
454
+
prot |= IVPU_MMU_ENTRY_FLAG_RO;
453
455
454
456
mutex_lock(&ctx->lock);
455
457
+1
-1
drivers/accel/ivpu/ivpu_mmu_context.h
+1
-1
drivers/accel/ivpu/ivpu_mmu_context.h
···
42
42
void ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node);
43
43
44
44
int ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
45
-
u64 vpu_addr, struct sg_table *sgt, bool llc_coherent);
45
+
u64 vpu_addr, struct sg_table *sgt, bool llc_coherent, bool read_only);
46
46
void ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
47
47
u64 vpu_addr, struct sg_table *sgt);
48
48
int ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
+52
include/uapi/drm/ivpu_accel.h
+52
include/uapi/drm/ivpu_accel.h
···
25
25
#define DRM_IVPU_CMDQ_CREATE 0x0b
26
26
#define DRM_IVPU_CMDQ_DESTROY 0x0c
27
27
#define DRM_IVPU_CMDQ_SUBMIT 0x0d
28
+
#define DRM_IVPU_BO_CREATE_FROM_USERPTR 0x0e
28
29
29
30
#define DRM_IOCTL_IVPU_GET_PARAM \
30
31
DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)
···
69
68
70
69
#define DRM_IOCTL_IVPU_CMDQ_SUBMIT \
71
70
DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_CMDQ_SUBMIT, struct drm_ivpu_cmdq_submit)
71
+
72
+
#define DRM_IOCTL_IVPU_BO_CREATE_FROM_USERPTR \
73
+
DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_CREATE_FROM_USERPTR, \
74
+
struct drm_ivpu_bo_create_from_userptr)
72
75
73
76
/**
74
77
* DOC: contexts
···
132
127
* command queue destroy and submit job on specific command queue.
133
128
*/
134
129
#define DRM_IVPU_CAP_MANAGE_CMDQ 3
130
+
/**
131
+
* DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR
132
+
*
133
+
* Driver supports creating buffer objects from user space memory pointers.
134
+
* This allows creating GEM buffers from existing user memory regions.
135
+
*/
136
+
#define DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR 4
135
137
136
138
/**
137
139
* struct drm_ivpu_param - Get/Set VPU parameters
···
206
194
#define DRM_IVPU_BO_HIGH_MEM DRM_IVPU_BO_SHAVE_MEM
207
195
#define DRM_IVPU_BO_MAPPABLE 0x00000002
208
196
#define DRM_IVPU_BO_DMA_MEM 0x00000004
197
+
#define DRM_IVPU_BO_READ_ONLY 0x00000008
209
198
210
199
#define DRM_IVPU_BO_CACHED 0x00000000
211
200
#define DRM_IVPU_BO_UNCACHED 0x00010000
···
217
204
(DRM_IVPU_BO_HIGH_MEM | \
218
205
DRM_IVPU_BO_MAPPABLE | \
219
206
DRM_IVPU_BO_DMA_MEM | \
207
+
DRM_IVPU_BO_READ_ONLY | \
220
208
DRM_IVPU_BO_CACHE_MASK)
221
209
222
210
/**
···
259
245
*
260
246
* Allocated BO will use write combining buffer for writes but reads will be
261
247
* uncached.
248
+
*/
249
+
__u32 flags;
250
+
251
+
/** @handle: Returned GEM object handle */
252
+
__u32 handle;
253
+
254
+
/** @vpu_addr: Returned VPU virtual address */
255
+
__u64 vpu_addr;
256
+
};
257
+
258
+
/**
259
+
* struct drm_ivpu_bo_create_from_userptr - Create dma-buf from user pointer
260
+
*
261
+
* Create a GEM buffer object from a user pointer to a memory region.
262
+
*/
263
+
struct drm_ivpu_bo_create_from_userptr {
264
+
/** @user_ptr: User pointer to memory region (must be page aligned) */
265
+
__u64 user_ptr;
266
+
267
+
/** @size: Size of the memory region in bytes (must be page aligned) */
268
+
__u64 size;
269
+
270
+
/**
271
+
* @flags:
272
+
*
273
+
* Supported flags:
274
+
*
275
+
* %DRM_IVPU_BO_HIGH_MEM:
276
+
*
277
+
* Allocate VPU address from >4GB range.
278
+
*
279
+
* %DRM_IVPU_BO_DMA_MEM:
280
+
*
281
+
* Allocate from DMA memory range accessible by hardware DMA.
282
+
*
283
+
* %DRM_IVPU_BO_READ_ONLY:
284
+
*
285
+
* Allocate as a read-only buffer object.
262
286
*/
263
287
__u32 flags;
264
288