Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
2/*
3 * Copyright © 2024 Intel Corporation
4 */
5
6#ifndef __DRM_GPUSVM_H__
7#define __DRM_GPUSVM_H__
8
9#include <linux/kref.h>
10#include <linux/interval_tree.h>
11#include <linux/mmu_notifier.h>
12
13struct dev_pagemap_ops;
14struct drm_device;
15struct drm_gpusvm;
16struct drm_gpusvm_notifier;
17struct drm_gpusvm_ops;
18struct drm_gpusvm_range;
19struct drm_pagemap;
20struct drm_pagemap_addr;
21
22/**
23 * struct drm_gpusvm_ops - Operations structure for GPU SVM
24 *
25 * This structure defines the operations for GPU Shared Virtual Memory (SVM).
26 * These operations are provided by the GPU driver to manage SVM ranges and
27 * notifiers.
28 */
29struct drm_gpusvm_ops {
30 /**
31 * @notifier_alloc: Allocate a GPU SVM notifier (optional)
32 *
33 * Allocate a GPU SVM notifier.
34 *
35 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure.
36 */
37 struct drm_gpusvm_notifier *(*notifier_alloc)(void);
38
39 /**
40 * @notifier_free: Free a GPU SVM notifier (optional)
41 * @notifier: Pointer to the GPU SVM notifier to be freed
42 *
43 * Free a GPU SVM notifier.
44 */
45 void (*notifier_free)(struct drm_gpusvm_notifier *notifier);
46
47 /**
48 * @range_alloc: Allocate a GPU SVM range (optional)
49 * @gpusvm: Pointer to the GPU SVM
50 *
51 * Allocate a GPU SVM range.
52 *
53 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure.
54 */
55 struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);
56
57 /**
58 * @range_free: Free a GPU SVM range (optional)
59 * @range: Pointer to the GPU SVM range to be freed
60 *
61 * Free a GPU SVM range.
62 */
63 void (*range_free)(struct drm_gpusvm_range *range);
64
65 /**
66 * @invalidate: Invalidate GPU SVM notifier (required)
67 * @gpusvm: Pointer to the GPU SVM
68 * @notifier: Pointer to the GPU SVM notifier
69 * @mmu_range: Pointer to the mmu_notifier_range structure
70 *
71 * Invalidate the GPU page tables. It can safely walk the notifier range
72 * RB tree/list in this function. Called while holding the notifier lock.
73 */
74 void (*invalidate)(struct drm_gpusvm *gpusvm,
75 struct drm_gpusvm_notifier *notifier,
76 const struct mmu_notifier_range *mmu_range);
77};
78
79/**
80 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier
81 *
82 * @gpusvm: Pointer to the GPU SVM structure
83 * @notifier: MMU interval notifier
84 * @itree: Interval tree node for the notifier (inserted in GPU SVM)
85 * @entry: List entry to fast interval tree traversal
86 * @root: Cached root node of the RB tree containing ranges
87 * @range_list: List head containing of ranges in the same order they appear in
88 * interval tree. This is useful to keep iterating ranges while
89 * doing modifications to RB tree.
90 * @flags: Flags for notifier
91 * @flags.removed: Flag indicating whether the MMU interval notifier has been
92 * removed
93 *
94 * This structure represents a GPU SVM notifier.
95 */
96struct drm_gpusvm_notifier {
97 struct drm_gpusvm *gpusvm;
98 struct mmu_interval_notifier notifier;
99 struct interval_tree_node itree;
100 struct list_head entry;
101 struct rb_root_cached root;
102 struct list_head range_list;
103 struct {
104 u32 removed : 1;
105 } flags;
106};
107
108/**
109 * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags
110 *
111 * @migrate_devmem: Flag indicating whether the pages can be migrated to device memory
112 * @unmapped: Flag indicating if the pages has been unmapped
113 * @partial_unmap: Flag indicating if the pages has been partially unmapped
114 * @has_devmem_pages: Flag indicating if the pages has devmem pages
115 * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
116 * @__flags: Flags for pages in u16 form (used for READ_ONCE)
117 */
118struct drm_gpusvm_pages_flags {
119 union {
120 struct {
121 /* All flags below must be set upon creation */
122 u16 migrate_devmem : 1;
123 /* All flags below must be set / cleared under notifier lock */
124 u16 unmapped : 1;
125 u16 partial_unmap : 1;
126 u16 has_devmem_pages : 1;
127 u16 has_dma_mapping : 1;
128 };
129 u16 __flags;
130 };
131};
132
133/**
134 * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
135 *
136 * @dma_addr: Device address array
137 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
138 * Note this is assuming only one drm_pagemap per range is allowed.
139 * @notifier_seq: Notifier sequence number of the range's pages
140 * @flags: Flags for range
141 * @flags.migrate_devmem: Flag indicating whether the range can be migrated to device memory
142 * @flags.unmapped: Flag indicating if the range has been unmapped
143 * @flags.partial_unmap: Flag indicating if the range has been partially unmapped
144 * @flags.has_devmem_pages: Flag indicating if the range has devmem pages
145 * @flags.has_dma_mapping: Flag indicating if the range has a DMA mapping
146 */
147struct drm_gpusvm_pages {
148 struct drm_pagemap_addr *dma_addr;
149 struct drm_pagemap *dpagemap;
150 unsigned long notifier_seq;
151 struct drm_gpusvm_pages_flags flags;
152};
153
154/**
155 * struct drm_gpusvm_range - Structure representing a GPU SVM range
156 *
157 * @gpusvm: Pointer to the GPU SVM structure
158 * @notifier: Pointer to the GPU SVM notifier
159 * @refcount: Reference count for the range
160 * @itree: Interval tree node for the range (inserted in GPU SVM notifier)
161 * @entry: List entry to fast interval tree traversal
162 * @pages: The pages for this range.
163 *
164 * This structure represents a GPU SVM range used for tracking memory ranges
165 * mapped in a DRM device.
166 */
167struct drm_gpusvm_range {
168 struct drm_gpusvm *gpusvm;
169 struct drm_gpusvm_notifier *notifier;
170 struct kref refcount;
171 struct interval_tree_node itree;
172 struct list_head entry;
173 struct drm_gpusvm_pages pages;
174};
175
176/**
177 * struct drm_gpusvm - GPU SVM structure
178 *
179 * @name: Name of the GPU SVM
180 * @drm: Pointer to the DRM device structure
181 * @mm: Pointer to the mm_struct for the address space
182 * @mm_start: Start address of GPU SVM
183 * @mm_range: Range of the GPU SVM
184 * @notifier_size: Size of individual notifiers
185 * @ops: Pointer to the operations structure for GPU SVM
186 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
187 * Entries should be powers of 2 in descending order.
188 * @num_chunks: Number of chunks
189 * @notifier_lock: Read-write semaphore for protecting notifier operations
190 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers
191 * @notifier_list: list head containing of notifiers in the same order they
192 * appear in interval tree. This is useful to keep iterating
193 * notifiers while doing modifications to RB tree.
194 *
195 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking
196 * memory ranges mapped in a DRM (Direct Rendering Manager) device.
197 *
198 * No reference counting is provided, as this is expected to be embedded in the
199 * driver VM structure along with the struct drm_gpuvm, which handles reference
200 * counting.
201 */
202struct drm_gpusvm {
203 const char *name;
204 struct drm_device *drm;
205 struct mm_struct *mm;
206 unsigned long mm_start;
207 unsigned long mm_range;
208 unsigned long notifier_size;
209 const struct drm_gpusvm_ops *ops;
210 const unsigned long *chunk_sizes;
211 int num_chunks;
212 struct rw_semaphore notifier_lock;
213 struct rb_root_cached root;
214 struct list_head notifier_list;
215#ifdef CONFIG_LOCKDEP
216 /**
217 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and
218 * drm_gpusvm_range_remove with a driver provided lock.
219 */
220 struct lockdep_map *lock_dep_map;
221#endif
222};
223
224/**
225 * struct drm_gpusvm_ctx - DRM GPU SVM context
226 *
227 * @device_private_page_owner: The device-private page owner to use for
228 * this operation
229 * @check_pages_threshold: Check CPU pages for present if chunk is less than or
230 * equal to threshold. If not present, reduce chunk
231 * size.
232 * @timeslice_ms: The timeslice MS which in minimum time a piece of memory
233 * remains with either exclusive GPU or CPU access.
234 * @in_notifier: entering from a MMU notifier
235 * @read_only: operating on read-only memory
236 * @devmem_possible: possible to use device memory
237 * @devmem_only: use only device memory
238 * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and
239 * single dpagemap is supported, mixing between multiple dpagemap
240 * is unsupported.
241 *
242 * Context that is DRM GPUSVM is operating in (i.e. user arguments).
243 */
244struct drm_gpusvm_ctx {
245 void *device_private_page_owner;
246 unsigned long check_pages_threshold;
247 unsigned long timeslice_ms;
248 unsigned int in_notifier :1;
249 unsigned int read_only :1;
250 unsigned int devmem_possible :1;
251 unsigned int devmem_only :1;
252 unsigned int allow_mixed :1;
253};
254
255int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
256 const char *name, struct drm_device *drm,
257 struct mm_struct *mm,
258 unsigned long mm_start, unsigned long mm_range,
259 unsigned long notifier_size,
260 const struct drm_gpusvm_ops *ops,
261 const unsigned long *chunk_sizes, int num_chunks);
262
263void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);
264
265void drm_gpusvm_free(struct drm_gpusvm *gpusvm);
266
267unsigned long
268drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
269 unsigned long start,
270 unsigned long end);
271
272struct drm_gpusvm_range *
273drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
274 unsigned long fault_addr,
275 unsigned long gpuva_start,
276 unsigned long gpuva_end,
277 const struct drm_gpusvm_ctx *ctx);
278
279void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
280 struct drm_gpusvm_range *range);
281
282int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
283 struct drm_gpusvm_range *range);
284
285struct drm_gpusvm_range *
286drm_gpusvm_range_get(struct drm_gpusvm_range *range);
287
288void drm_gpusvm_range_put(struct drm_gpusvm_range *range);
289
290bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
291 struct drm_gpusvm_range *range);
292
293int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
294 struct drm_gpusvm_range *range,
295 const struct drm_gpusvm_ctx *ctx);
296
297void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
298 struct drm_gpusvm_range *range,
299 const struct drm_gpusvm_ctx *ctx);
300
301bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
302 unsigned long end);
303
304struct drm_gpusvm_notifier *
305drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
306 unsigned long end);
307
308struct drm_gpusvm_range *
309drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
310 unsigned long end);
311
312void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
313 const struct mmu_notifier_range *mmu_range);
314
315int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
316 struct drm_gpusvm_pages *svm_pages,
317 struct mm_struct *mm,
318 struct mmu_interval_notifier *notifier,
319 unsigned long pages_start, unsigned long pages_end,
320 const struct drm_gpusvm_ctx *ctx);
321
322void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
323 struct drm_gpusvm_pages *svm_pages,
324 unsigned long npages,
325 const struct drm_gpusvm_ctx *ctx);
326
327void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
328 struct drm_gpusvm_pages *svm_pages,
329 unsigned long npages);
330
331/**
332 * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
333 * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
334 * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as
335 * the @pagemap argument to the drm_gpusvm_scan_mm() function.
336 * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is
337 * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to
338 * the same device private owner.
339 * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages.
340 * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least
341 * two different struct dev_pagemaps. All pages belong to the same device private
342 * owner.
343 * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages
344 * and device-private pages. All device-private pages belong to the same device
345 * private owner.
346 */
347enum drm_gpusvm_scan_result {
348 DRM_GPUSVM_SCAN_UNPOPULATED,
349 DRM_GPUSVM_SCAN_EQUAL,
350 DRM_GPUSVM_SCAN_OTHER,
351 DRM_GPUSVM_SCAN_SYSTEM,
352 DRM_GPUSVM_SCAN_MIXED_DEVICE,
353 DRM_GPUSVM_SCAN_MIXED,
354};
355
356enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
357 void *dev_private_owner,
358 const struct dev_pagemap *pagemap);
359
360#ifdef CONFIG_LOCKDEP
361/**
362 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM
363 * @gpusvm: Pointer to the GPU SVM structure.
364 * @lock: the lock used to protect the gpuva list. The locking primitive
365 * must contain a dep_map field.
366 *
367 * Call this to annotate drm_gpusvm_range_find_or_insert and
368 * drm_gpusvm_range_remove.
369 */
370#define drm_gpusvm_driver_set_lock(gpusvm, lock) \
371 do { \
372 if (!WARN((gpusvm)->lock_dep_map, \
373 "GPUSVM range lock should be set only once."))\
374 (gpusvm)->lock_dep_map = &(lock)->dep_map; \
375 } while (0)
376#else
377#define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
378#endif
379
380/**
381 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier
382 * @gpusvm__: Pointer to the GPU SVM structure.
383 *
384 * Abstract client usage GPU SVM notifier lock, take lock
385 */
386#define drm_gpusvm_notifier_lock(gpusvm__) \
387 down_read(&(gpusvm__)->notifier_lock)
388
389/**
390 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier
391 * @gpusvm__: Pointer to the GPU SVM structure.
392 *
393 * Abstract client usage GPU SVM notifier lock, drop lock
394 */
395#define drm_gpusvm_notifier_unlock(gpusvm__) \
396 up_read(&(gpusvm__)->notifier_lock)
397
398/**
399 * drm_gpusvm_range_start() - GPU SVM range start address
400 * @range: Pointer to the GPU SVM range
401 *
402 * Return: GPU SVM range start address
403 */
404static inline unsigned long
405drm_gpusvm_range_start(struct drm_gpusvm_range *range)
406{
407 return range->itree.start;
408}
409
410/**
411 * drm_gpusvm_range_end() - GPU SVM range end address
412 * @range: Pointer to the GPU SVM range
413 *
414 * Return: GPU SVM range end address
415 */
416static inline unsigned long
417drm_gpusvm_range_end(struct drm_gpusvm_range *range)
418{
419 return range->itree.last + 1;
420}
421
422/**
423 * drm_gpusvm_range_size() - GPU SVM range size
424 * @range: Pointer to the GPU SVM range
425 *
426 * Return: GPU SVM range size
427 */
428static inline unsigned long
429drm_gpusvm_range_size(struct drm_gpusvm_range *range)
430{
431 return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
432}
433
434/**
435 * drm_gpusvm_notifier_start() - GPU SVM notifier start address
436 * @notifier: Pointer to the GPU SVM notifier
437 *
438 * Return: GPU SVM notifier start address
439 */
440static inline unsigned long
441drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
442{
443 return notifier->itree.start;
444}
445
446/**
447 * drm_gpusvm_notifier_end() - GPU SVM notifier end address
448 * @notifier: Pointer to the GPU SVM notifier
449 *
450 * Return: GPU SVM notifier end address
451 */
452static inline unsigned long
453drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
454{
455 return notifier->itree.last + 1;
456}
457
458/**
459 * drm_gpusvm_notifier_size() - GPU SVM notifier size
460 * @notifier: Pointer to the GPU SVM notifier
461 *
462 * Return: GPU SVM notifier size
463 */
464static inline unsigned long
465drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
466{
467 return drm_gpusvm_notifier_end(notifier) -
468 drm_gpusvm_notifier_start(notifier);
469}
470
471/**
472 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list
473 * @range: a pointer to the current GPU SVM range
474 *
475 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the
476 * current range is the last one or if the input range is NULL.
477 */
478static inline struct drm_gpusvm_range *
479__drm_gpusvm_range_next(struct drm_gpusvm_range *range)
480{
481 if (range && !list_is_last(&range->entry,
482 &range->notifier->range_list))
483 return list_next_entry(range, entry);
484
485 return NULL;
486}
487
488/**
489 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier
490 * @range__: Iterator variable for the ranges. If set, it indicates the start of
491 * the iterator. If NULL, call drm_gpusvm_range_find() to get the range.
492 * @notifier__: Pointer to the GPU SVM notifier
493 * @start__: Start address of the range
494 * @end__: End address of the range
495 *
496 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe
497 * to use while holding the driver SVM lock or the notifier lock.
498 */
499#define drm_gpusvm_for_each_range(range__, notifier__, start__, end__) \
500 for ((range__) = (range__) ?: \
501 drm_gpusvm_range_find((notifier__), (start__), (end__)); \
502 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \
503 (range__) = __drm_gpusvm_range_next(range__))
504
505/**
506 * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier
507 * @range__: Iterator variable for the ranges
508 * @next__: Iterator variable for the ranges temporay storage
509 * @notifier__: Pointer to the GPU SVM notifier
510 * @start__: Start address of the range
511 * @end__: End address of the range
512 *
513 * This macro is used to iterate over GPU SVM ranges in a notifier while
514 * removing ranges from it.
515 */
516#define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__) \
517 for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)), \
518 (next__) = __drm_gpusvm_range_next(range__); \
519 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \
520 (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))
521
522/**
523 * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list
524 * @notifier: a pointer to the current drm_gpusvm_notifier
525 *
526 * Return: A pointer to the next drm_gpusvm_notifier if available, or NULL if
527 * the current notifier is the last one or if the input notifier is
528 * NULL.
529 */
530static inline struct drm_gpusvm_notifier *
531__drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
532{
533 if (notifier && !list_is_last(¬ifier->entry,
534 ¬ifier->gpusvm->notifier_list))
535 return list_next_entry(notifier, entry);
536
537 return NULL;
538}
539
540/**
541 * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm
542 * @notifier__: Iterator variable for the notifiers
543 * @gpusvm__: Pointer to the GPU SVM notifier
544 * @start__: Start address of the notifier
545 * @end__: End address of the notifier
546 *
547 * This macro is used to iterate over GPU SVM notifiers in a gpusvm.
548 */
549#define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__) \
550 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)); \
551 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \
552 (notifier__) = __drm_gpusvm_notifier_next(notifier__))
553
554/**
555 * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm
556 * @notifier__: Iterator variable for the notifiers
557 * @next__: Iterator variable for the notifiers temporay storage
558 * @gpusvm__: Pointer to the GPU SVM notifier
559 * @start__: Start address of the notifier
560 * @end__: End address of the notifier
561 *
562 * This macro is used to iterate over GPU SVM notifiers in a gpusvm while
563 * removing notifiers from it.
564 */
565#define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__) \
566 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)), \
567 (next__) = __drm_gpusvm_notifier_next(notifier__); \
568 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \
569 (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))
570
571#endif /* __DRM_GPUSVM_H__ */