Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
drm/xe/svm: Add svm ranges migration policy on atomic access
If the platform does not support atomic access on system memory, and the
ranges are in system memory, but the user requires atomic accesses on
the VMA, then migrate the ranges to VRAM. Apply this policy for prefetch
operations as well.
v2
- Drop unnecessary vm_dbg
v3 (Matthew Brost)
- fix atomic policy
- prefetch shouldn't have any impact of atomic
- bo can be accessed from vma, avoid duplicate parameter
v4 (Matthew Brost)
- Remove TODO comment
- Fix comment
- Dont allow gpu atomic ops when user is setting atomic attr as CPU
v5 (Matthew Brost)
- Fix atomic checks
- Add userptr checks
Cc: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: Matthew Brost <matthew.brost@intel.com>
Link: https://lore.kernel.org/r/20250821173104.3030148-10-himal.prasad.ghimiray@intel.com
Signed-off-by: Himal Prasad Ghimiray <himal.prasad.ghimiray@intel.com>
+99
-30
+13
-10
drivers/gpu/drm/xe/xe_pt.c
+13
-10
drivers/gpu/drm/xe/xe_pt.c
···
640
640
* - In all other cases device atomics will be disabled with AE=0 until an application
641
641
* request differently using a ioctl like madvise.
642
642
*/
643
-
static bool xe_atomic_for_vram(struct xe_vm *vm)
643
+
static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma)
644
644
{
645
+
if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
646
+
return false;
647
+
645
648
return true;
646
649
}
647
650
648
-
static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_bo *bo)
651
+
static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma)
649
652
{
650
653
struct xe_device *xe = vm->xe;
654
+
struct xe_bo *bo = xe_vma_bo(vma);
651
655
652
-
if (!xe->info.has_device_atomics_on_smem)
656
+
if (!xe->info.has_device_atomics_on_smem ||
657
+
vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
653
658
return false;
659
+
660
+
if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE)
661
+
return true;
654
662
655
663
/*
656
664
* If a SMEM+LMEM allocation is backed by SMEM, a device
657
665
* atomics will cause a gpu page fault and which then
658
666
* gets migrated to LMEM, bind such allocations with
659
667
* device atomics enabled.
660
-
*
661
-
* TODO: Revisit this. Perhaps add something like a
662
-
* fault_on_atomics_in_system UAPI flag.
663
-
* Note that this also prohibits GPU atomics in LR mode for
664
-
* userptr and system memory on DGFX.
665
668
*/
666
669
return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) ||
667
670
(bo && xe_bo_has_single_placement(bo))));
···
747
744
goto walk_pt;
748
745
749
746
if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
750
-
xe_walk.default_vram_pte = xe_atomic_for_vram(vm) ? XE_USM_PPGTT_PTE_AE : 0;
751
-
xe_walk.default_system_pte = xe_atomic_for_system(vm, bo) ?
747
+
xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0;
748
+
xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ?
752
749
XE_USM_PPGTT_PTE_AE : 0;
753
750
}
754
751
+31
-19
drivers/gpu/drm/xe/xe_svm.c
+31
-19
drivers/gpu/drm/xe/xe_svm.c
···
789
789
return true;
790
790
}
791
791
792
-
/**
793
-
* xe_svm_handle_pagefault() - SVM handle page fault
794
-
* @vm: The VM.
795
-
* @vma: The CPU address mirror VMA.
796
-
* @gt: The gt upon the fault occurred.
797
-
* @fault_addr: The GPU fault address.
798
-
* @atomic: The fault atomic access bit.
799
-
*
800
-
* Create GPU bindings for a SVM page fault. Optionally migrate to device
801
-
* memory.
802
-
*
803
-
* Return: 0 on success, negative error code on error.
804
-
*/
805
-
int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
806
-
struct xe_gt *gt, u64 fault_addr,
807
-
bool atomic)
792
+
static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
793
+
struct xe_gt *gt, u64 fault_addr,
794
+
bool need_vram)
808
795
{
809
796
struct drm_gpusvm_ctx ctx = {
810
797
.read_only = xe_vma_read_only(vma),
···
799
812
IS_ENABLED(CONFIG_DRM_XE_PAGEMAP),
800
813
.check_pages_threshold = IS_DGFX(vm->xe) &&
801
814
IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) ? SZ_64K : 0,
802
-
.devmem_only = atomic && IS_DGFX(vm->xe) &&
803
-
IS_ENABLED(CONFIG_DRM_XE_PAGEMAP),
804
-
.timeslice_ms = atomic && IS_DGFX(vm->xe) &&
815
+
.devmem_only = need_vram && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP),
816
+
.timeslice_ms = need_vram && IS_DGFX(vm->xe) &&
805
817
IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) ?
806
818
vm->xe->atomic_svm_timeslice_ms : 0,
807
819
};
···
901
915
err_out:
902
916
903
917
return err;
918
+
}
919
+
920
+
/**
921
+
* xe_svm_handle_pagefault() - SVM handle page fault
922
+
* @vm: The VM.
923
+
* @vma: The CPU address mirror VMA.
924
+
* @gt: The gt upon the fault occurred.
925
+
* @fault_addr: The GPU fault address.
926
+
* @atomic: The fault atomic access bit.
927
+
*
928
+
* Create GPU bindings for a SVM page fault. Optionally migrate to device
929
+
* memory.
930
+
*
931
+
* Return: 0 on success, negative error code on error.
932
+
*/
933
+
int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
934
+
struct xe_gt *gt, u64 fault_addr,
935
+
bool atomic)
936
+
{
937
+
int need_vram;
938
+
939
+
need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
940
+
if (need_vram < 0)
941
+
return need_vram;
942
+
943
+
return __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, need_vram ? true : false);
904
944
}
905
945
906
946
/**
+39
drivers/gpu/drm/xe/xe_vm.c
+39
drivers/gpu/drm/xe/xe_vm.c
···
4213
4213
}
4214
4214
4215
4215
/**
4216
+
* xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations
4217
+
* @xe: Pointer to the XE device structure
4218
+
* @vma: Pointer to the virtual memory area (VMA) structure
4219
+
* @is_atomic: In pagefault path and atomic operation
4220
+
*
4221
+
* This function determines whether the given VMA needs to be migrated to
4222
+
* VRAM in order to do atomic GPU operation.
4223
+
*
4224
+
* Return:
4225
+
* 1 - Migration to VRAM is required
4226
+
* 0 - Migration is not required
4227
+
* -EACCES - Invalid access for atomic memory attr
4228
+
*
4229
+
*/
4230
+
int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic)
4231
+
{
4232
+
if (!IS_DGFX(xe) || !is_atomic)
4233
+
return 0;
4234
+
4235
+
/*
4236
+
* NOTE: The checks implemented here are platform-specific. For
4237
+
* instance, on a device supporting CXL atomics, these would ideally
4238
+
* work universally without additional handling.
4239
+
*/
4240
+
switch (vma->attr.atomic_access) {
4241
+
case DRM_XE_ATOMIC_DEVICE:
4242
+
return !xe->info.has_device_atomics_on_smem;
4243
+
4244
+
case DRM_XE_ATOMIC_CPU:
4245
+
return -EACCES;
4246
+
4247
+
case DRM_XE_ATOMIC_UNDEFINED:
4248
+
case DRM_XE_ATOMIC_GLOBAL:
4249
+
default:
4250
+
return 1;
4251
+
}
4252
+
}
4253
+
4254
+
/**
4216
4255
* xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops
4217
4256
* @vm: Pointer to the xe_vm structure
4218
4257
* @start: Starting input address
+2
drivers/gpu/drm/xe/xe_vm.h
+2
drivers/gpu/drm/xe/xe_vm.h
···
171
171
172
172
struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr);
173
173
174
+
int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic);
175
+
174
176
int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t addr, uint64_t size);
175
177
176
178
/**
+14
-1
drivers/gpu/drm/xe/xe_vm_madvise.c
+14
-1
drivers/gpu/drm/xe/xe_vm_madvise.c
···
85
85
struct xe_vma **vmas, int num_vmas,
86
86
struct drm_xe_madvise *op)
87
87
{
88
-
/* Implementation pending */
88
+
int i;
89
+
90
+
xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC);
91
+
xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU);
92
+
93
+
for (i = 0; i < num_vmas; i++) {
94
+
if ((xe_vma_is_userptr(vmas[i]) &&
95
+
!(op->atomic.val == DRM_XE_ATOMIC_DEVICE &&
96
+
xe->info.has_device_atomics_on_smem)))
97
+
continue;
98
+
99
+
vmas[i]->attr.atomic_access = op->atomic.val;
100
+
/*TODO: handle bo backed vmas */
101
+
}
89
102
}
90
103
91
104
static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm,