Merge tag 'drm-xe-next-2025-12-30' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next

+2 -1

drivers/gpu/drm/Makefile

··· 109 109 110 110 drm_gpusvm_helper-y := \ 111 111 drm_gpusvm.o\ 112 - drm_pagemap.o 112 + drm_pagemap.o\ 113 + drm_pagemap_util.o 113 114 obj-$(CONFIG_DRM_GPUSVM) += drm_gpusvm_helper.o 114 115 115 116 obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o

+124

drivers/gpu/drm/drm_gpusvm.c

··· 744 744 } 745 745 746 746 /** 747 + * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range 748 + * @range: Pointer to the struct drm_gpusvm_range to check. 749 + * @dev_private_owner: The struct dev_private_owner to use to determine 750 + * compatible device-private pages. 751 + * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific 752 + * checks. 753 + * 754 + * Scan the CPU address space corresponding to @range and return the 755 + * current migration state. Note that the result may be invalid as 756 + * soon as the function returns. It's an advisory check. 757 + * 758 + * TODO: Bail early and call hmm_range_fault() for subranges. 759 + * 760 + * Return: See &enum drm_gpusvm_scan_result. 761 + */ 762 + enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, 763 + void *dev_private_owner, 764 + const struct dev_pagemap *pagemap) 765 + { 766 + struct mmu_interval_notifier *notifier = &range->notifier->notifier; 767 + unsigned long start = drm_gpusvm_range_start(range); 768 + unsigned long end = drm_gpusvm_range_end(range); 769 + struct hmm_range hmm_range = { 770 + .default_flags = 0, 771 + .notifier = notifier, 772 + .start = start, 773 + .end = end, 774 + .dev_private_owner = dev_private_owner, 775 + }; 776 + unsigned long timeout = 777 + jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 778 + enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state; 779 + unsigned long *pfns; 780 + unsigned long npages = npages_in_range(start, end); 781 + const struct dev_pagemap *other = NULL; 782 + int err, i; 783 + 784 + pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 785 + if (!pfns) 786 + return DRM_GPUSVM_SCAN_UNPOPULATED; 787 + 788 + hmm_range.hmm_pfns = pfns; 789 + 790 + retry: 791 + hmm_range.notifier_seq = mmu_interval_read_begin(notifier); 792 + mmap_read_lock(range->gpusvm->mm); 793 + 794 + while (true) { 795 + err = hmm_range_fault(&hmm_range); 796 + if (err == -EBUSY) { 797 + if (time_after(jiffies, timeout)) 798 + break; 799 + 800 + hmm_range.notifier_seq = 801 + mmu_interval_read_begin(notifier); 802 + continue; 803 + } 804 + break; 805 + } 806 + mmap_read_unlock(range->gpusvm->mm); 807 + if (err) 808 + goto err_free; 809 + 810 + drm_gpusvm_notifier_lock(range->gpusvm); 811 + if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { 812 + drm_gpusvm_notifier_unlock(range->gpusvm); 813 + goto retry; 814 + } 815 + 816 + for (i = 0; i < npages;) { 817 + struct page *page; 818 + const struct dev_pagemap *cur = NULL; 819 + 820 + if (!(pfns[i] & HMM_PFN_VALID)) { 821 + state = DRM_GPUSVM_SCAN_UNPOPULATED; 822 + goto err_free; 823 + } 824 + 825 + page = hmm_pfn_to_page(pfns[i]); 826 + if (is_device_private_page(page) || 827 + is_device_coherent_page(page)) 828 + cur = page_pgmap(page); 829 + 830 + if (cur == pagemap) { 831 + new_state = DRM_GPUSVM_SCAN_EQUAL; 832 + } else if (cur && (cur == other || !other)) { 833 + new_state = DRM_GPUSVM_SCAN_OTHER; 834 + other = cur; 835 + } else if (cur) { 836 + new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 837 + } else { 838 + new_state = DRM_GPUSVM_SCAN_SYSTEM; 839 + } 840 + 841 + /* 842 + * TODO: Could use an array for state 843 + * transitions, and caller might want it 844 + * to bail early for some results. 845 + */ 846 + if (state == DRM_GPUSVM_SCAN_UNPOPULATED) { 847 + state = new_state; 848 + } else if (state != new_state) { 849 + if (new_state == DRM_GPUSVM_SCAN_SYSTEM || 850 + state == DRM_GPUSVM_SCAN_SYSTEM) 851 + state = DRM_GPUSVM_SCAN_MIXED; 852 + else if (state != DRM_GPUSVM_SCAN_MIXED) 853 + state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 854 + } 855 + 856 + i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 857 + } 858 + 859 + err_free: 860 + drm_gpusvm_notifier_unlock(range->gpusvm); 861 + 862 + kvfree(pfns); 863 + return state; 864 + } 865 + EXPORT_SYMBOL(drm_gpusvm_scan_mm); 866 + 867 + /** 747 868 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range 748 869 * @gpusvm: Pointer to the GPU SVM structure 749 870 * @notifier: Pointer to the GPU SVM notifier structure ··· 1159 1038 flags.has_dma_mapping = false; 1160 1039 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1161 1040 1041 + drm_pagemap_put(svm_pages->dpagemap); 1162 1042 svm_pages->dpagemap = NULL; 1163 1043 } 1164 1044 } ··· 1556 1434 1557 1435 if (pagemap) { 1558 1436 flags.has_devmem_pages = true; 1437 + drm_pagemap_get(dpagemap); 1438 + drm_pagemap_put(svm_pages->dpagemap); 1559 1439 svm_pages->dpagemap = dpagemap; 1560 1440 } 1561 1441

+480 -83

drivers/gpu/drm/drm_pagemap.c

··· 3 3 * Copyright © 2024-2025 Intel Corporation 4 4 */ 5 5 6 + #include <linux/dma-fence.h> 6 7 #include <linux/dma-mapping.h> 7 8 #include <linux/migrate.h> 8 9 #include <linux/pagemap.h> 9 10 #include <drm/drm_drv.h> 10 11 #include <drm/drm_pagemap.h> 12 + #include <drm/drm_pagemap_util.h> 13 + #include <drm/drm_print.h> 11 14 12 15 /** 13 16 * DOC: Overview ··· 65 62 * 66 63 * @refcount: Reference count for the zdd 67 64 * @devmem_allocation: device memory allocation 68 - * @device_private_page_owner: Device private pages owner 65 + * @dpagemap: Refcounted pointer to the underlying struct drm_pagemap. 69 66 * 70 67 * This structure serves as a generic wrapper installed in 71 68 * page->zone_device_data. It provides infrastructure for looking up a device ··· 77 74 struct drm_pagemap_zdd { 78 75 struct kref refcount; 79 76 struct drm_pagemap_devmem *devmem_allocation; 80 - void *device_private_page_owner; 77 + struct drm_pagemap *dpagemap; 81 78 }; 82 79 83 80 /** 84 81 * drm_pagemap_zdd_alloc() - Allocate a zdd structure. 85 - * @device_private_page_owner: Device private pages owner 82 + * @dpagemap: Pointer to the underlying struct drm_pagemap. 86 83 * 87 84 * This function allocates and initializes a new zdd structure. It sets up the 88 85 * reference count and initializes the destroy work. ··· 90 87 * Return: Pointer to the allocated zdd on success, ERR_PTR() on failure. 91 88 */ 92 89 static struct drm_pagemap_zdd * 93 - drm_pagemap_zdd_alloc(void *device_private_page_owner) 90 + drm_pagemap_zdd_alloc(struct drm_pagemap *dpagemap) 94 91 { 95 92 struct drm_pagemap_zdd *zdd; 96 93 ··· 100 97 101 98 kref_init(&zdd->refcount); 102 99 zdd->devmem_allocation = NULL; 103 - zdd->device_private_page_owner = device_private_page_owner; 100 + zdd->dpagemap = drm_pagemap_get(dpagemap); 104 101 105 102 return zdd; 106 103 } ··· 130 127 struct drm_pagemap_zdd *zdd = 131 128 container_of(ref, struct drm_pagemap_zdd, refcount); 132 129 struct drm_pagemap_devmem *devmem = zdd->devmem_allocation; 130 + struct drm_pagemap *dpagemap = zdd->dpagemap; 133 131 134 132 if (devmem) { 135 133 complete_all(&devmem->detached); ··· 138 134 devmem->ops->devmem_release(devmem); 139 135 } 140 136 kfree(zdd); 137 + drm_pagemap_put(dpagemap); 141 138 } 142 139 143 140 /** ··· 206 201 207 202 /** 208 203 * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration 209 - * @dev: The device for which the pages are being mapped 210 - * @pagemap_addr: Array to store DMA information corresponding to mapped pages 211 - * @migrate_pfn: Array of migrate page frame numbers to map 212 - * @npages: Number of pages to map 204 + * @dev: The device performing the migration. 205 + * @local_dpagemap: The drm_pagemap local to the migrating device. 206 + * @pagemap_addr: Array to store DMA information corresponding to mapped pages. 207 + * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map. 208 + * @npages: Number of system pages or peer pages to map. 213 209 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) 210 + * @mdetails: Details governing the migration behaviour. 214 211 * 215 212 * This function maps pages of memory for migration usage in GPU SVM. It 216 213 * iterates over each page frame number provided in @migrate_pfn, maps the ··· 222 215 * Returns: 0 on success, -EFAULT if an error occurs during mapping. 223 216 */ 224 217 static int drm_pagemap_migrate_map_pages(struct device *dev, 218 + struct drm_pagemap *local_dpagemap, 225 219 struct drm_pagemap_addr *pagemap_addr, 226 220 unsigned long *migrate_pfn, 227 221 unsigned long npages, 228 - enum dma_data_direction dir) 222 + enum dma_data_direction dir, 223 + const struct drm_pagemap_migrate_details *mdetails) 229 224 { 230 - unsigned long i; 225 + unsigned long num_peer_pages = 0, num_local_pages = 0, i; 231 226 232 227 for (i = 0; i < npages;) { 233 228 struct page *page = migrate_pfn_to_page(migrate_pfn[i]); ··· 240 231 if (!page) 241 232 goto next; 242 233 243 - if (WARN_ON_ONCE(is_zone_device_page(page))) 244 - return -EFAULT; 245 - 246 234 folio = page_folio(page); 247 235 order = folio_order(folio); 248 236 249 - dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); 250 - if (dma_mapping_error(dev, dma_addr)) 251 - return -EFAULT; 237 + if (is_device_private_page(page)) { 238 + struct drm_pagemap_zdd *zdd = page->zone_device_data; 239 + struct drm_pagemap *dpagemap = zdd->dpagemap; 240 + struct drm_pagemap_addr addr; 252 241 253 - pagemap_addr[i] = 254 - drm_pagemap_addr_encode(dma_addr, 255 - DRM_INTERCONNECT_SYSTEM, 256 - order, dir); 242 + if (dpagemap == local_dpagemap) { 243 + if (!mdetails->can_migrate_same_pagemap) 244 + goto next; 245 + 246 + num_local_pages += NR_PAGES(order); 247 + } else { 248 + num_peer_pages += NR_PAGES(order); 249 + } 250 + 251 + addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir); 252 + if (dma_mapping_error(dev, addr.addr)) 253 + return -EFAULT; 254 + 255 + pagemap_addr[i] = addr; 256 + } else { 257 + dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); 258 + if (dma_mapping_error(dev, dma_addr)) 259 + return -EFAULT; 260 + 261 + pagemap_addr[i] = 262 + drm_pagemap_addr_encode(dma_addr, 263 + DRM_INTERCONNECT_SYSTEM, 264 + order, dir); 265 + } 257 266 258 267 next: 259 268 i += NR_PAGES(order); 260 269 } 270 + 271 + if (num_peer_pages) 272 + drm_dbg(local_dpagemap->drm, "Migrating %lu peer pages over interconnect.\n", 273 + num_peer_pages); 274 + if (num_local_pages) 275 + drm_dbg(local_dpagemap->drm, "Migrating %lu local pages over interconnect.\n", 276 + num_local_pages); 261 277 262 278 return 0; 263 279 } ··· 290 256 /** 291 257 * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration 292 258 * @dev: The device for which the pages were mapped 259 + * @migrate_pfn: Array of migrate pfns set up for the mapped pages. Used to 260 + * determine the drm_pagemap of a peer device private page. 293 261 * @pagemap_addr: Array of DMA information corresponding to mapped pages 294 262 * @npages: Number of pages to unmap 295 263 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) ··· 302 266 */ 303 267 static void drm_pagemap_migrate_unmap_pages(struct device *dev, 304 268 struct drm_pagemap_addr *pagemap_addr, 269 + unsigned long *migrate_pfn, 305 270 unsigned long npages, 306 271 enum dma_data_direction dir) 307 272 { 308 273 unsigned long i; 309 274 310 275 for (i = 0; i < npages;) { 311 - if (!pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) 276 + struct page *page = migrate_pfn_to_page(migrate_pfn[i]); 277 + 278 + if (!page || !pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) 312 279 goto next; 313 280 314 - dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE << pagemap_addr[i].order, dir); 281 + if (is_zone_device_page(page)) { 282 + struct drm_pagemap_zdd *zdd = page->zone_device_data; 283 + struct drm_pagemap *dpagemap = zdd->dpagemap; 284 + 285 + dpagemap->ops->device_unmap(dpagemap, dev, pagemap_addr[i]); 286 + } else { 287 + dma_unmap_page(dev, pagemap_addr[i].addr, 288 + PAGE_SIZE << pagemap_addr[i].order, dir); 289 + } 315 290 316 291 next: 317 292 i += NR_PAGES(pagemap_addr[i].order); ··· 335 288 return (end - start) >> PAGE_SHIFT; 336 289 } 337 290 291 + static int 292 + drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem, 293 + struct device *remote_device, 294 + struct drm_pagemap *remote_dpagemap, 295 + unsigned long local_pfns[], 296 + struct page *remote_pages[], 297 + struct drm_pagemap_addr pagemap_addr[], 298 + unsigned long npages, 299 + const struct drm_pagemap_devmem_ops *ops, 300 + const struct drm_pagemap_migrate_details *mdetails) 301 + 302 + { 303 + int err = drm_pagemap_migrate_map_pages(remote_device, remote_dpagemap, 304 + pagemap_addr, local_pfns, 305 + npages, DMA_FROM_DEVICE, mdetails); 306 + 307 + if (err) 308 + goto out; 309 + 310 + err = ops->copy_to_ram(remote_pages, pagemap_addr, npages, 311 + devmem->pre_migrate_fence); 312 + out: 313 + drm_pagemap_migrate_unmap_pages(remote_device, pagemap_addr, local_pfns, 314 + npages, DMA_FROM_DEVICE); 315 + return err; 316 + } 317 + 318 + static int 319 + drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem, 320 + unsigned long sys_pfns[], 321 + struct page *local_pages[], 322 + struct drm_pagemap_addr pagemap_addr[], 323 + unsigned long npages, 324 + const struct drm_pagemap_devmem_ops *ops, 325 + const struct drm_pagemap_migrate_details *mdetails) 326 + { 327 + int err = drm_pagemap_migrate_map_pages(devmem->dev, devmem->dpagemap, 328 + pagemap_addr, sys_pfns, npages, 329 + DMA_TO_DEVICE, mdetails); 330 + 331 + if (err) 332 + goto out; 333 + 334 + err = ops->copy_to_devmem(local_pages, pagemap_addr, npages, 335 + devmem->pre_migrate_fence); 336 + out: 337 + drm_pagemap_migrate_unmap_pages(devmem->dev, pagemap_addr, sys_pfns, npages, 338 + DMA_TO_DEVICE); 339 + return err; 340 + } 341 + 342 + /** 343 + * struct migrate_range_loc - Cursor into the loop over migrate_pfns for migrating to 344 + * device. 345 + * @start: The current loop index. 346 + * @device: migrating device. 347 + * @dpagemap: Pointer to struct drm_pagemap used by the migrating device. 348 + * @ops: The copy ops to be used for the migrating device. 349 + */ 350 + struct migrate_range_loc { 351 + unsigned long start; 352 + struct device *device; 353 + struct drm_pagemap *dpagemap; 354 + const struct drm_pagemap_devmem_ops *ops; 355 + }; 356 + 357 + static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem, 358 + unsigned long src_pfns[], 359 + unsigned long dst_pfns[], 360 + struct page *pages[], 361 + struct drm_pagemap_addr pagemap_addr[], 362 + struct migrate_range_loc *last, 363 + const struct migrate_range_loc *cur, 364 + const struct drm_pagemap_migrate_details *mdetails) 365 + { 366 + int ret = 0; 367 + 368 + if (cur->start == 0) 369 + goto out; 370 + 371 + if (cur->start <= last->start) 372 + return 0; 373 + 374 + if (cur->dpagemap == last->dpagemap && cur->ops == last->ops) 375 + return 0; 376 + 377 + if (last->dpagemap) 378 + ret = drm_pagemap_migrate_remote_to_local(devmem, 379 + last->device, 380 + last->dpagemap, 381 + &dst_pfns[last->start], 382 + &pages[last->start], 383 + &pagemap_addr[last->start], 384 + cur->start - last->start, 385 + last->ops, mdetails); 386 + 387 + else 388 + ret = drm_pagemap_migrate_sys_to_dev(devmem, 389 + &src_pfns[last->start], 390 + &pages[last->start], 391 + &pagemap_addr[last->start], 392 + cur->start - last->start, 393 + last->ops, mdetails); 394 + 395 + out: 396 + *last = *cur; 397 + return ret; 398 + } 399 + 338 400 /** 339 401 * drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory 340 402 * @devmem_allocation: The device memory allocation to migrate to. 341 403 * The caller should hold a reference to the device memory allocation, 342 - * and the reference is consumed by this function unless it returns with 404 + * and the reference is consumed by this function even if it returns with 343 405 * an error. 344 406 * @mm: Pointer to the struct mm_struct. 345 407 * @start: Start of the virtual address range to migrate. 346 408 * @end: End of the virtual address range to migrate. 347 - * @timeslice_ms: The time requested for the migrated pagemap pages to 348 - * be present in @mm before being allowed to be migrated back. 349 - * @pgmap_owner: Not used currently, since only system memory is considered. 409 + * @mdetails: Details to govern the migration. 350 410 * 351 411 * This function migrates the specified virtual address range to device memory. 352 412 * It performs the necessary setup and invokes the driver-specific operations for ··· 471 317 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, 472 318 struct mm_struct *mm, 473 319 unsigned long start, unsigned long end, 474 - unsigned long timeslice_ms, 475 - void *pgmap_owner) 320 + const struct drm_pagemap_migrate_details *mdetails) 476 321 { 477 322 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; 323 + struct drm_pagemap *dpagemap = devmem_allocation->dpagemap; 324 + struct dev_pagemap *pagemap = dpagemap->pagemap; 478 325 struct migrate_vma migrate = { 479 326 .start = start, 480 327 .end = end, 481 - .pgmap_owner = pgmap_owner, 482 - .flags = MIGRATE_VMA_SELECT_SYSTEM, 328 + .pgmap_owner = pagemap->owner, 329 + .flags = MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_DEVICE_COHERENT | 330 + MIGRATE_VMA_SELECT_DEVICE_PRIVATE, 483 331 }; 484 332 unsigned long i, npages = npages_in_range(start, end); 333 + unsigned long own_pages = 0, migrated_pages = 0; 334 + struct migrate_range_loc cur, last = {.device = dpagemap->drm->dev, .ops = ops}; 485 335 struct vm_area_struct *vas; 486 336 struct drm_pagemap_zdd *zdd = NULL; 487 337 struct page **pages; ··· 524 366 pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); 525 367 pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; 526 368 527 - zdd = drm_pagemap_zdd_alloc(pgmap_owner); 369 + zdd = drm_pagemap_zdd_alloc(dpagemap); 528 370 if (!zdd) { 529 371 err = -ENOMEM; 530 - goto err_free; 372 + kvfree(buf); 373 + goto err_out; 531 374 } 375 + zdd->devmem_allocation = devmem_allocation; /* Owns ref */ 532 376 533 377 migrate.vma = vas; 534 378 migrate.src = buf; ··· 541 381 goto err_free; 542 382 543 383 if (!migrate.cpages) { 544 - err = -EFAULT; 384 + /* No pages to migrate. Raced or unknown device pages. */ 385 + err = -EBUSY; 545 386 goto err_free; 546 387 } 547 388 548 389 if (migrate.cpages != npages) { 390 + /* 391 + * Some pages to migrate. But we want to migrate all or 392 + * nothing. Raced or unknown device pages. 393 + */ 549 394 err = -EBUSY; 550 - goto err_finalize; 395 + goto err_aborted_migration; 396 + } 397 + 398 + /* Count device-private pages to migrate */ 399 + for (i = 0; i < npages;) { 400 + struct page *src_page = migrate_pfn_to_page(migrate.src[i]); 401 + unsigned long nr_pages = src_page ? NR_PAGES(folio_order(page_folio(src_page))) : 1; 402 + 403 + if (src_page && is_zone_device_page(src_page)) { 404 + if (page_pgmap(src_page) == pagemap) 405 + own_pages += nr_pages; 406 + } 407 + 408 + i += nr_pages; 409 + } 410 + 411 + drm_dbg(dpagemap->drm, "Total pages %lu; Own pages: %lu.\n", 412 + npages, own_pages); 413 + if (own_pages == npages) { 414 + err = 0; 415 + drm_dbg(dpagemap->drm, "Migration wasn't necessary.\n"); 416 + goto err_aborted_migration; 417 + } else if (own_pages && !mdetails->can_migrate_same_pagemap) { 418 + err = -EBUSY; 419 + drm_dbg(dpagemap->drm, "Migration aborted due to fragmentation.\n"); 420 + goto err_aborted_migration; 551 421 } 552 422 553 423 err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst); 554 424 if (err) 555 425 goto err_finalize; 556 426 557 - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, 558 - migrate.src, npages, DMA_TO_DEVICE); 559 - 560 - if (err) 561 - goto err_finalize; 427 + own_pages = 0; 562 428 563 429 for (i = 0; i < npages; ++i) { 564 430 struct page *page = pfn_to_page(migrate.dst[i]); 431 + struct page *src_page = migrate_pfn_to_page(migrate.src[i]); 432 + cur.start = i; 565 433 566 - pages[i] = page; 434 + pages[i] = NULL; 435 + if (src_page && is_device_private_page(src_page)) { 436 + struct drm_pagemap_zdd *src_zdd = src_page->zone_device_data; 437 + 438 + if (page_pgmap(src_page) == pagemap && 439 + !mdetails->can_migrate_same_pagemap) { 440 + migrate.dst[i] = 0; 441 + own_pages++; 442 + continue; 443 + } 444 + if (mdetails->source_peer_migrates) { 445 + cur.dpagemap = src_zdd->dpagemap; 446 + cur.ops = src_zdd->devmem_allocation->ops; 447 + cur.device = cur.dpagemap->drm->dev; 448 + pages[i] = src_page; 449 + } 450 + } 451 + if (!pages[i]) { 452 + cur.dpagemap = NULL; 453 + cur.ops = ops; 454 + cur.device = dpagemap->drm->dev; 455 + pages[i] = page; 456 + } 567 457 migrate.dst[i] = migrate_pfn(migrate.dst[i]); 568 458 drm_pagemap_get_devmem_page(page, zdd); 569 - } 570 459 571 - err = ops->copy_to_devmem(pages, pagemap_addr, npages); 460 + /* If we switched the migrating drm_pagemap, migrate previous pages now */ 461 + err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, 462 + pages, pagemap_addr, &last, &cur, 463 + mdetails); 464 + if (err) 465 + goto err_finalize; 466 + } 467 + cur.start = npages; 468 + cur.ops = NULL; /* Force migration */ 469 + err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, 470 + pages, pagemap_addr, &last, &cur, mdetails); 572 471 if (err) 573 472 goto err_finalize; 574 473 474 + drm_WARN_ON(dpagemap->drm, !!own_pages); 475 + 476 + dma_fence_put(devmem_allocation->pre_migrate_fence); 477 + devmem_allocation->pre_migrate_fence = NULL; 478 + 575 479 /* Upon success bind devmem allocation to range and zdd */ 576 480 devmem_allocation->timeslice_expiration = get_jiffies_64() + 577 - msecs_to_jiffies(timeslice_ms); 578 - zdd->devmem_allocation = devmem_allocation; /* Owns ref */ 481 + msecs_to_jiffies(mdetails->timeslice_ms); 579 482 580 483 err_finalize: 581 484 if (err) 582 485 drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); 486 + err_aborted_migration: 583 487 migrate_vma_pages(&migrate); 488 + 489 + for (i = 0; i < npages;) { 490 + struct page *page = migrate_pfn_to_page(migrate.src[i]); 491 + unsigned long nr_pages = page ? NR_PAGES(folio_order(page_folio(page))) : 1; 492 + 493 + if (migrate.src[i] & MIGRATE_PFN_MIGRATE) 494 + migrated_pages += nr_pages; 495 + 496 + i += nr_pages; 497 + } 498 + 499 + if (!err && migrated_pages < npages - own_pages) { 500 + drm_dbg(dpagemap->drm, "Raced while finalizing migration.\n"); 501 + err = -EBUSY; 502 + } 503 + 584 504 migrate_vma_finalize(&migrate); 585 - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, 586 - DMA_TO_DEVICE); 587 505 err_free: 588 - if (zdd) 589 - drm_pagemap_zdd_put(zdd); 506 + drm_pagemap_zdd_put(zdd); 590 507 kvfree(buf); 508 + return err; 509 + 591 510 err_out: 511 + devmem_allocation->ops->devmem_release(devmem_allocation); 592 512 return err; 593 513 } 594 514 EXPORT_SYMBOL_GPL(drm_pagemap_migrate_to_devmem); ··· 778 538 return -ENOMEM; 779 539 } 780 540 541 + static void drm_pagemap_dev_unhold_work(struct work_struct *work); 542 + static LLIST_HEAD(drm_pagemap_unhold_list); 543 + static DECLARE_WORK(drm_pagemap_work, drm_pagemap_dev_unhold_work); 544 + 545 + /** 546 + * struct drm_pagemap_dev_hold - Struct to aid in drm_device release. 547 + * @link: Link into drm_pagemap_unhold_list for deferred reference releases. 548 + * @drm: drm device to put. 549 + * 550 + * When a struct drm_pagemap is released, we also need to release the 551 + * reference it holds on the drm device. However, typically that needs 552 + * to be done separately from a system-wide workqueue. 553 + * Each time a struct drm_pagemap is initialized 554 + * (or re-initialized if cached) therefore allocate a separate 555 + * drm_pagemap_dev_hold item, from which we put the drm device and 556 + * associated module. 557 + */ 558 + struct drm_pagemap_dev_hold { 559 + struct llist_node link; 560 + struct drm_device *drm; 561 + }; 562 + 563 + static void drm_pagemap_release(struct kref *ref) 564 + { 565 + struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref); 566 + struct drm_pagemap_dev_hold *dev_hold = dpagemap->dev_hold; 567 + 568 + /* 569 + * We know the pagemap provider is alive at this point, since 570 + * the struct drm_pagemap_dev_hold holds a reference to the 571 + * pagemap provider drm_device and its module. 572 + */ 573 + dpagemap->dev_hold = NULL; 574 + drm_pagemap_shrinker_add(dpagemap); 575 + llist_add(&dev_hold->link, &drm_pagemap_unhold_list); 576 + schedule_work(&drm_pagemap_work); 577 + /* 578 + * Here, either the provider device is still alive, since if called from 579 + * page_free(), the caller is holding a reference on the dev_pagemap, 580 + * or if called from drm_pagemap_put(), the direct caller is still alive. 581 + * This ensures we can't race with THIS module unload. 582 + */ 583 + } 584 + 585 + static void drm_pagemap_dev_unhold_work(struct work_struct *work) 586 + { 587 + struct llist_node *node = llist_del_all(&drm_pagemap_unhold_list); 588 + struct drm_pagemap_dev_hold *dev_hold, *next; 589 + 590 + /* 591 + * Deferred release of drm_pagemap provider device and module. 592 + * THIS module is kept alive during the release by the 593 + * flush_work() in the drm_pagemap_exit() function. 594 + */ 595 + llist_for_each_entry_safe(dev_hold, next, node, link) { 596 + struct drm_device *drm = dev_hold->drm; 597 + struct module *module = drm->driver->fops->owner; 598 + 599 + drm_dbg(drm, "Releasing reference on provider device and module.\n"); 600 + drm_dev_put(drm); 601 + module_put(module); 602 + kfree(dev_hold); 603 + } 604 + } 605 + 606 + static struct drm_pagemap_dev_hold * 607 + drm_pagemap_dev_hold(struct drm_pagemap *dpagemap) 608 + { 609 + struct drm_pagemap_dev_hold *dev_hold; 610 + struct drm_device *drm = dpagemap->drm; 611 + 612 + dev_hold = kzalloc(sizeof(*dev_hold), GFP_KERNEL); 613 + if (!dev_hold) 614 + return ERR_PTR(-ENOMEM); 615 + 616 + init_llist_node(&dev_hold->link); 617 + dev_hold->drm = drm; 618 + (void)try_module_get(drm->driver->fops->owner); 619 + drm_dev_get(drm); 620 + 621 + return dev_hold; 622 + } 623 + 624 + /** 625 + * drm_pagemap_reinit() - Reinitialize a drm_pagemap 626 + * @dpagemap: The drm_pagemap to reinitialize 627 + * 628 + * Reinitialize a drm_pagemap, for which drm_pagemap_release 629 + * has already been called. This interface is intended for the 630 + * situation where the driver caches a destroyed drm_pagemap. 631 + * 632 + * Return: 0 on success, negative error code on failure. 633 + */ 634 + int drm_pagemap_reinit(struct drm_pagemap *dpagemap) 635 + { 636 + dpagemap->dev_hold = drm_pagemap_dev_hold(dpagemap); 637 + if (IS_ERR(dpagemap->dev_hold)) 638 + return PTR_ERR(dpagemap->dev_hold); 639 + 640 + kref_init(&dpagemap->ref); 641 + return 0; 642 + } 643 + EXPORT_SYMBOL(drm_pagemap_reinit); 644 + 645 + /** 646 + * drm_pagemap_init() - Initialize a pre-allocated drm_pagemap 647 + * @dpagemap: The drm_pagemap to initialize. 648 + * @pagemap: The associated dev_pagemap providing the device 649 + * private pages. 650 + * @drm: The drm device. The drm_pagemap holds a reference on the 651 + * drm_device and the module owning the drm_device until 652 + * drm_pagemap_release(). This facilitates drm_pagemap exporting. 653 + * @ops: The drm_pagemap ops. 654 + * 655 + * Initialize and take an initial reference on a drm_pagemap. 656 + * After successful return, use drm_pagemap_put() to destroy. 657 + * 658 + ** Return: 0 on success, negative error code on error. 659 + */ 660 + int drm_pagemap_init(struct drm_pagemap *dpagemap, 661 + struct dev_pagemap *pagemap, 662 + struct drm_device *drm, 663 + const struct drm_pagemap_ops *ops) 664 + { 665 + kref_init(&dpagemap->ref); 666 + dpagemap->ops = ops; 667 + dpagemap->pagemap = pagemap; 668 + dpagemap->drm = drm; 669 + dpagemap->cache = NULL; 670 + INIT_LIST_HEAD(&dpagemap->shrink_link); 671 + 672 + return drm_pagemap_reinit(dpagemap); 673 + } 674 + EXPORT_SYMBOL(drm_pagemap_init); 675 + 676 + /** 677 + * drm_pagemap_put() - Put a struct drm_pagemap reference 678 + * @dpagemap: Pointer to a struct drm_pagemap object. 679 + * 680 + * Puts a struct drm_pagemap reference and frees the drm_pagemap object 681 + * if the refount reaches zero. 682 + */ 683 + void drm_pagemap_put(struct drm_pagemap *dpagemap) 684 + { 685 + if (likely(dpagemap)) { 686 + drm_pagemap_shrinker_might_lock(dpagemap); 687 + kref_put(&dpagemap->ref, drm_pagemap_release); 688 + } 689 + } 690 + EXPORT_SYMBOL(drm_pagemap_put); 691 + 781 692 /** 782 693 * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM 783 694 * @devmem_allocation: Pointer to the device memory allocation ··· 941 550 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) 942 551 { 943 552 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; 553 + struct drm_pagemap_migrate_details mdetails = {}; 944 554 unsigned long npages, mpages = 0; 945 555 struct page **pages; 946 556 unsigned long *src, *dst; ··· 980 588 if (err || !mpages) 981 589 goto err_finalize; 982 590 983 - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, 984 - dst, npages, DMA_FROM_DEVICE); 591 + err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, 592 + devmem_allocation->dpagemap, pagemap_addr, 593 + dst, npages, DMA_FROM_DEVICE, 594 + &mdetails); 985 595 if (err) 986 596 goto err_finalize; 987 597 988 598 for (i = 0; i < npages; ++i) 989 599 pages[i] = migrate_pfn_to_page(src[i]); 990 600 991 - err = ops->copy_to_ram(pages, pagemap_addr, npages); 601 + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 992 602 if (err) 993 603 goto err_finalize; 994 604 ··· 999 605 drm_pagemap_migration_unlock_put_pages(npages, dst); 1000 606 migrate_device_pages(src, dst, npages); 1001 607 migrate_device_finalize(src, dst, npages); 1002 - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, 608 + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages, 1003 609 DMA_FROM_DEVICE); 610 + 1004 611 err_free: 1005 612 kvfree(buf); 1006 613 err_out: ··· 1022 627 /** 1023 628 * __drm_pagemap_migrate_to_ram() - Migrate GPU SVM range to RAM (internal) 1024 629 * @vas: Pointer to the VM area structure 1025 - * @device_private_page_owner: Device private pages owner 1026 - * @page: Pointer to the page for fault handling (can be NULL) 630 + * @page: Pointer to the page for fault handling. 1027 631 * @fault_addr: Fault address 1028 632 * @size: Size of migration 1029 633 * ··· 1033 639 * Return: 0 on success, negative error code on failure. 1034 640 */ 1035 641 static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, 1036 - void *device_private_page_owner, 1037 642 struct page *page, 1038 643 unsigned long fault_addr, 1039 644 unsigned long size) 1040 645 { 1041 646 struct migrate_vma migrate = { 1042 647 .vma = vas, 1043 - .pgmap_owner = device_private_page_owner, 648 + .pgmap_owner = page_pgmap(page)->owner, 1044 649 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE | 1045 650 MIGRATE_VMA_SELECT_DEVICE_COHERENT, 1046 651 .fault_page = page, 1047 652 }; 653 + struct drm_pagemap_migrate_details mdetails = {}; 1048 654 struct drm_pagemap_zdd *zdd; 1049 655 const struct drm_pagemap_devmem_ops *ops; 1050 656 struct device *dev = NULL; ··· 1055 661 void *buf; 1056 662 int i, err = 0; 1057 663 1058 - if (page) { 1059 - zdd = page->zone_device_data; 1060 - if (time_before64(get_jiffies_64(), 1061 - zdd->devmem_allocation->timeslice_expiration)) 1062 - return 0; 1063 - } 664 + zdd = page->zone_device_data; 665 + if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration)) 666 + return 0; 1064 667 1065 668 start = ALIGN_DOWN(fault_addr, size); 1066 669 end = ALIGN(fault_addr + 1, size); ··· 1093 702 if (!migrate.cpages) 1094 703 goto err_free; 1095 704 1096 - if (!page) { 1097 - for (i = 0; i < npages; ++i) { 1098 - if (!(migrate.src[i] & MIGRATE_PFN_MIGRATE)) 1099 - continue; 1100 - 1101 - page = migrate_pfn_to_page(migrate.src[i]); 1102 - break; 1103 - } 1104 - 1105 - if (!page) 1106 - goto err_finalize; 1107 - } 1108 - zdd = page->zone_device_data; 1109 705 ops = zdd->devmem_allocation->ops; 1110 706 dev = zdd->devmem_allocation->dev; 1111 707 ··· 1102 724 if (err) 1103 725 goto err_finalize; 1104 726 1105 - err = drm_pagemap_migrate_map_pages(dev, pagemap_addr, migrate.dst, npages, 1106 - DMA_FROM_DEVICE); 727 + err = drm_pagemap_migrate_map_pages(dev, zdd->dpagemap, pagemap_addr, migrate.dst, npages, 728 + DMA_FROM_DEVICE, &mdetails); 1107 729 if (err) 1108 730 goto err_finalize; 1109 731 1110 732 for (i = 0; i < npages; ++i) 1111 733 pages[i] = migrate_pfn_to_page(migrate.src[i]); 1112 734 1113 - err = ops->copy_to_ram(pages, pagemap_addr, npages); 735 + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 1114 736 if (err) 1115 737 goto err_finalize; 1116 738 ··· 1120 742 migrate_vma_pages(&migrate); 1121 743 migrate_vma_finalize(&migrate); 1122 744 if (dev) 1123 - drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages, 1124 - DMA_FROM_DEVICE); 745 + drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, migrate.dst, 746 + npages, DMA_FROM_DEVICE); 1125 747 err_free: 1126 748 kvfree(buf); 1127 749 err_out: ··· 1158 780 int err; 1159 781 1160 782 err = __drm_pagemap_migrate_to_ram(vmf->vma, 1161 - zdd->device_private_page_owner, 1162 783 vmf->page, vmf->address, 1163 784 zdd->devmem_allocation->size); 1164 785 ··· 1190 813 * @ops: Pointer to the operations structure for GPU SVM device memory 1191 814 * @dpagemap: The struct drm_pagemap we're allocating from. 1192 815 * @size: Size of device memory allocation 816 + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 817 + * (May be NULL). 1193 818 */ 1194 819 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 1195 820 struct device *dev, struct mm_struct *mm, 1196 821 const struct drm_pagemap_devmem_ops *ops, 1197 - struct drm_pagemap *dpagemap, size_t size) 822 + struct drm_pagemap *dpagemap, size_t size, 823 + struct dma_fence *pre_migrate_fence) 1198 824 { 1199 825 init_completion(&devmem_allocation->detached); 1200 826 devmem_allocation->dev = dev; ··· 1205 825 devmem_allocation->ops = ops; 1206 826 devmem_allocation->dpagemap = dpagemap; 1207 827 devmem_allocation->size = size; 828 + devmem_allocation->pre_migrate_fence = pre_migrate_fence; 1208 829 } 1209 830 EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init); 1210 831 ··· 1261 880 return err; 1262 881 } 1263 882 EXPORT_SYMBOL(drm_pagemap_populate_mm); 883 + 884 + void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim) 885 + { 886 + if (dpagemap->ops->destroy) 887 + dpagemap->ops->destroy(dpagemap, is_atomic_or_reclaim); 888 + else 889 + kfree(dpagemap); 890 + } 891 + 892 + static void drm_pagemap_exit(void) 893 + { 894 + flush_work(&drm_pagemap_work); 895 + if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list))) 896 + disable_work_sync(&drm_pagemap_work); 897 + } 898 + module_exit(drm_pagemap_exit);

+568

drivers/gpu/drm/drm_pagemap_util.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #include <linux/slab.h> 7 + 8 + #include <drm/drm_drv.h> 9 + #include <drm/drm_managed.h> 10 + #include <drm/drm_pagemap.h> 11 + #include <drm/drm_pagemap_util.h> 12 + #include <drm/drm_print.h> 13 + 14 + /** 15 + * struct drm_pagemap_cache - Lookup structure for pagemaps 16 + * 17 + * Structure to keep track of active (refcount > 1) and inactive 18 + * (refcount == 0) pagemaps. Inactive pagemaps can be made active 19 + * again by waiting for the @queued completion (indicating that the 20 + * pagemap has been put on the @shrinker's list of shrinkable 21 + * pagemaps, and then successfully removing it from @shrinker's 22 + * list. The latter may fail if the shrinker is already in the 23 + * process of freeing the pagemap. A struct drm_pagemap_cache can 24 + * hold a single struct drm_pagemap. 25 + */ 26 + struct drm_pagemap_cache { 27 + /** @lookup_mutex: Mutex making the lookup process atomic */ 28 + struct mutex lookup_mutex; 29 + /** @lock: Lock protecting the @dpagemap pointer */ 30 + spinlock_t lock; 31 + /** @shrinker: Pointer to the shrinker used for this cache. Immutable. */ 32 + struct drm_pagemap_shrinker *shrinker; 33 + /** @dpagemap: Non-refcounted pointer to the drm_pagemap */ 34 + struct drm_pagemap *dpagemap; 35 + /** 36 + * @queued: Signals when an inactive drm_pagemap has been put on 37 + * @shrinker's list. 38 + */ 39 + struct completion queued; 40 + }; 41 + 42 + /** 43 + * struct drm_pagemap_shrinker - Shrinker to remove unused pagemaps 44 + */ 45 + struct drm_pagemap_shrinker { 46 + /** @drm: Pointer to the drm device. */ 47 + struct drm_device *drm; 48 + /** @lock: Spinlock to protect the @dpagemaps list. */ 49 + spinlock_t lock; 50 + /** @dpagemaps: List of unused dpagemaps. */ 51 + struct list_head dpagemaps; 52 + /** @num_dpagemaps: Number of unused dpagemaps in @dpagemaps. */ 53 + atomic_t num_dpagemaps; 54 + /** @shrink: Pointer to the struct shrinker. */ 55 + struct shrinker *shrink; 56 + }; 57 + 58 + static bool drm_pagemap_shrinker_cancel(struct drm_pagemap *dpagemap); 59 + 60 + static void drm_pagemap_cache_fini(void *arg) 61 + { 62 + struct drm_pagemap_cache *cache = arg; 63 + struct drm_pagemap *dpagemap; 64 + 65 + drm_dbg(cache->shrinker->drm, "Destroying dpagemap cache.\n"); 66 + spin_lock(&cache->lock); 67 + dpagemap = cache->dpagemap; 68 + if (!dpagemap) { 69 + spin_unlock(&cache->lock); 70 + goto out; 71 + } 72 + 73 + if (drm_pagemap_shrinker_cancel(dpagemap)) { 74 + cache->dpagemap = NULL; 75 + spin_unlock(&cache->lock); 76 + drm_pagemap_destroy(dpagemap, false); 77 + } 78 + 79 + out: 80 + mutex_destroy(&cache->lookup_mutex); 81 + kfree(cache); 82 + } 83 + 84 + /** 85 + * drm_pagemap_cache_create_devm() - Create a drm_pagemap_cache 86 + * @shrinker: Pointer to a struct drm_pagemap_shrinker. 87 + * 88 + * Create a device-managed drm_pagemap cache. The cache is automatically 89 + * destroyed on struct device removal, at which point any *inactive* 90 + * drm_pagemap's are destroyed. 91 + * 92 + * Return: Pointer to a struct drm_pagemap_cache on success. Error pointer 93 + * on failure. 94 + */ 95 + struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct drm_pagemap_shrinker *shrinker) 96 + { 97 + struct drm_pagemap_cache *cache = kzalloc(sizeof(*cache), GFP_KERNEL); 98 + int err; 99 + 100 + if (!cache) 101 + return ERR_PTR(-ENOMEM); 102 + 103 + mutex_init(&cache->lookup_mutex); 104 + spin_lock_init(&cache->lock); 105 + cache->shrinker = shrinker; 106 + init_completion(&cache->queued); 107 + err = devm_add_action_or_reset(shrinker->drm->dev, drm_pagemap_cache_fini, cache); 108 + if (err) 109 + return ERR_PTR(err); 110 + 111 + return cache; 112 + } 113 + EXPORT_SYMBOL(drm_pagemap_cache_create_devm); 114 + 115 + /** 116 + * DOC: Cache lookup 117 + * 118 + * Cache lookup should be done under a locked mutex, so that a 119 + * failed drm_pagemap_get_from_cache() and a following 120 + * drm_pagemap_cache_setpagemap() are carried out as an atomic 121 + * operation WRT other lookups. Otherwise, racing lookups may 122 + * unnecessarily concurrently create pagemaps to fulfill a 123 + * failed lookup. The API provides two functions to perform this lock, 124 + * drm_pagemap_lock_lookup() and drm_pagemap_unlock_lookup() and they 125 + * should be used in the following way: 126 + * 127 + * .. code-block:: c 128 + * 129 + * drm_pagemap_lock_lookup(cache); 130 + * dpagemap = drm_pagemap_get_from_cache(cache); 131 + * if (dpagemap) 132 + * goto out_unlock; 133 + * 134 + * dpagemap = driver_create_new_dpagemap(); 135 + * if (!IS_ERR(dpagemap)) 136 + * drm_pagemap_cache_set_pagemap(cache, dpagemap); 137 + * 138 + * out_unlock: 139 + * drm_pagemap_unlock_lookup(cache); 140 + */ 141 + 142 + /** 143 + * drm_pagemap_cache_lock_lookup() - Lock a drm_pagemap_cache for lookup. 144 + * @cache: The drm_pagemap_cache to lock. 145 + * 146 + * Return: %-EINTR if interrupted while blocking. %0 otherwise. 147 + */ 148 + int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache *cache) 149 + { 150 + return mutex_lock_interruptible(&cache->lookup_mutex); 151 + } 152 + EXPORT_SYMBOL(drm_pagemap_cache_lock_lookup); 153 + 154 + /** 155 + * drm_pagemap_cache_unlock_lookup() - Unlock a drm_pagemap_cache after lookup. 156 + * @cache: The drm_pagemap_cache to unlock. 157 + */ 158 + void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache *cache) 159 + { 160 + mutex_unlock(&cache->lookup_mutex); 161 + } 162 + EXPORT_SYMBOL(drm_pagemap_cache_unlock_lookup); 163 + 164 + /** 165 + * drm_pagemap_get_from_cache() - Lookup of drm_pagemaps. 166 + * @cache: The cache used for lookup. 167 + * 168 + * If an active pagemap is present in the cache, it is immediately returned. 169 + * If an inactive pagemap is present, it's removed from the shrinker list and 170 + * an attempt is made to make it active. 171 + * If no pagemap present or the attempt to make it active failed, %NULL is returned 172 + * to indicate to the caller to create a new drm_pagemap and insert it into 173 + * the cache. 174 + * 175 + * Return: A reference-counted pointer to a drm_pagemap if successful. An error 176 + * pointer if an error occurred, or %NULL if no drm_pagemap was found and 177 + * the caller should insert a new one. 178 + */ 179 + struct drm_pagemap *drm_pagemap_get_from_cache(struct drm_pagemap_cache *cache) 180 + { 181 + struct drm_pagemap *dpagemap; 182 + int err; 183 + 184 + lockdep_assert_held(&cache->lookup_mutex); 185 + retry: 186 + spin_lock(&cache->lock); 187 + dpagemap = cache->dpagemap; 188 + if (drm_pagemap_get_unless_zero(dpagemap)) { 189 + spin_unlock(&cache->lock); 190 + return dpagemap; 191 + } 192 + 193 + if (!dpagemap) { 194 + spin_unlock(&cache->lock); 195 + return NULL; 196 + } 197 + 198 + if (!try_wait_for_completion(&cache->queued)) { 199 + spin_unlock(&cache->lock); 200 + err = wait_for_completion_interruptible(&cache->queued); 201 + if (err) 202 + return ERR_PTR(err); 203 + goto retry; 204 + } 205 + 206 + if (drm_pagemap_shrinker_cancel(dpagemap)) { 207 + cache->dpagemap = NULL; 208 + spin_unlock(&cache->lock); 209 + err = drm_pagemap_reinit(dpagemap); 210 + if (err) { 211 + drm_pagemap_destroy(dpagemap, false); 212 + return ERR_PTR(err); 213 + } 214 + drm_pagemap_cache_set_pagemap(cache, dpagemap); 215 + } else { 216 + cache->dpagemap = NULL; 217 + spin_unlock(&cache->lock); 218 + dpagemap = NULL; 219 + } 220 + 221 + return dpagemap; 222 + } 223 + EXPORT_SYMBOL(drm_pagemap_get_from_cache); 224 + 225 + /** 226 + * drm_pagemap_cache_set_pagemap() - Assign a drm_pagemap to a drm_pagemap_cache 227 + * @cache: The cache to assign the drm_pagemap to. 228 + * @dpagemap: The drm_pagemap to assign. 229 + * 230 + * The function must be called to populate a drm_pagemap_cache only 231 + * after a call to drm_pagemap_get_from_cache() returns NULL. 232 + */ 233 + void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache *cache, struct drm_pagemap *dpagemap) 234 + { 235 + struct drm_device *drm = dpagemap->drm; 236 + 237 + lockdep_assert_held(&cache->lookup_mutex); 238 + spin_lock(&cache->lock); 239 + dpagemap->cache = cache; 240 + swap(cache->dpagemap, dpagemap); 241 + reinit_completion(&cache->queued); 242 + spin_unlock(&cache->lock); 243 + drm_WARN_ON(drm, !!dpagemap); 244 + } 245 + EXPORT_SYMBOL(drm_pagemap_cache_set_pagemap); 246 + 247 + /** 248 + * drm_pagemap_get_from_cache_if_active() - Quick lookup of active drm_pagemaps 249 + * @cache: The cache to lookup from. 250 + * 251 + * Function that should be used to lookup a drm_pagemap that is already active. 252 + * (refcount > 0). 253 + * 254 + * Return: A pointer to the cache's drm_pagemap if it's active; %NULL otherwise. 255 + */ 256 + struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct drm_pagemap_cache *cache) 257 + { 258 + struct drm_pagemap *dpagemap; 259 + 260 + spin_lock(&cache->lock); 261 + dpagemap = drm_pagemap_get_unless_zero(cache->dpagemap); 262 + spin_unlock(&cache->lock); 263 + 264 + return dpagemap; 265 + } 266 + EXPORT_SYMBOL(drm_pagemap_get_from_cache_if_active); 267 + 268 + static bool drm_pagemap_shrinker_cancel(struct drm_pagemap *dpagemap) 269 + { 270 + struct drm_pagemap_cache *cache = dpagemap->cache; 271 + struct drm_pagemap_shrinker *shrinker = cache->shrinker; 272 + 273 + spin_lock(&shrinker->lock); 274 + if (list_empty(&dpagemap->shrink_link)) { 275 + spin_unlock(&shrinker->lock); 276 + return false; 277 + } 278 + 279 + list_del_init(&dpagemap->shrink_link); 280 + atomic_dec(&shrinker->num_dpagemaps); 281 + spin_unlock(&shrinker->lock); 282 + return true; 283 + } 284 + 285 + #ifdef CONFIG_PROVE_LOCKING 286 + /** 287 + * drm_pagemap_shrinker_might_lock() - lockdep test for drm_pagemap_shrinker_add() 288 + * @dpagemap: The drm pagemap. 289 + * 290 + * The drm_pagemap_shrinker_add() function performs some locking. 291 + * This function can be called in code-paths that might 292 + * call drm_pagemap_shrinker_add() to detect any lockdep problems early. 293 + */ 294 + void drm_pagemap_shrinker_might_lock(struct drm_pagemap *dpagemap) 295 + { 296 + int idx; 297 + 298 + if (drm_dev_enter(dpagemap->drm, &idx)) { 299 + struct drm_pagemap_cache *cache = dpagemap->cache; 300 + 301 + if (cache) 302 + might_lock(&cache->shrinker->lock); 303 + 304 + drm_dev_exit(idx); 305 + } 306 + } 307 + #endif 308 + 309 + /** 310 + * drm_pagemap_shrinker_add() - Add a drm_pagemap to the shrinker list or destroy 311 + * @dpagemap: The drm_pagemap. 312 + * 313 + * If @dpagemap is associated with a &struct drm_pagemap_cache AND the 314 + * struct device backing the drm device is still alive, add @dpagemap to 315 + * the &struct drm_pagemap_shrinker list of shrinkable drm_pagemaps. 316 + * 317 + * Otherwise destroy the pagemap directly using drm_pagemap_destroy(). 318 + * 319 + * This is an internal function which is not intended to be exposed to drivers. 320 + */ 321 + void drm_pagemap_shrinker_add(struct drm_pagemap *dpagemap) 322 + { 323 + struct drm_pagemap_cache *cache; 324 + struct drm_pagemap_shrinker *shrinker; 325 + int idx; 326 + 327 + /* 328 + * The pagemap cache and shrinker are disabled at 329 + * pci device remove time. After that, dpagemaps 330 + * are freed directly. 331 + */ 332 + if (!drm_dev_enter(dpagemap->drm, &idx)) 333 + goto out_no_cache; 334 + 335 + cache = dpagemap->cache; 336 + if (!cache) { 337 + drm_dev_exit(idx); 338 + goto out_no_cache; 339 + } 340 + 341 + shrinker = cache->shrinker; 342 + spin_lock(&shrinker->lock); 343 + list_add_tail(&dpagemap->shrink_link, &shrinker->dpagemaps); 344 + atomic_inc(&shrinker->num_dpagemaps); 345 + spin_unlock(&shrinker->lock); 346 + complete_all(&cache->queued); 347 + drm_dev_exit(idx); 348 + return; 349 + 350 + out_no_cache: 351 + drm_pagemap_destroy(dpagemap, true); 352 + } 353 + 354 + static unsigned long 355 + drm_pagemap_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 356 + { 357 + struct drm_pagemap_shrinker *shrinker = shrink->private_data; 358 + unsigned long count = atomic_read(&shrinker->num_dpagemaps); 359 + 360 + return count ? : SHRINK_EMPTY; 361 + } 362 + 363 + static unsigned long 364 + drm_pagemap_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 365 + { 366 + struct drm_pagemap_shrinker *shrinker = shrink->private_data; 367 + struct drm_pagemap *dpagemap; 368 + struct drm_pagemap_cache *cache; 369 + unsigned long nr_freed = 0; 370 + 371 + sc->nr_scanned = 0; 372 + spin_lock(&shrinker->lock); 373 + do { 374 + dpagemap = list_first_entry_or_null(&shrinker->dpagemaps, typeof(*dpagemap), 375 + shrink_link); 376 + if (!dpagemap) 377 + break; 378 + 379 + atomic_dec(&shrinker->num_dpagemaps); 380 + list_del_init(&dpagemap->shrink_link); 381 + spin_unlock(&shrinker->lock); 382 + 383 + sc->nr_scanned++; 384 + nr_freed++; 385 + 386 + cache = dpagemap->cache; 387 + spin_lock(&cache->lock); 388 + cache->dpagemap = NULL; 389 + spin_unlock(&cache->lock); 390 + 391 + drm_dbg(dpagemap->drm, "Shrinking dpagemap %p.\n", dpagemap); 392 + drm_pagemap_destroy(dpagemap, true); 393 + spin_lock(&shrinker->lock); 394 + } while (sc->nr_scanned < sc->nr_to_scan); 395 + spin_unlock(&shrinker->lock); 396 + 397 + return sc->nr_scanned ? nr_freed : SHRINK_STOP; 398 + } 399 + 400 + static void drm_pagemap_shrinker_fini(void *arg) 401 + { 402 + struct drm_pagemap_shrinker *shrinker = arg; 403 + 404 + drm_dbg(shrinker->drm, "Destroying dpagemap shrinker.\n"); 405 + drm_WARN_ON(shrinker->drm, !!atomic_read(&shrinker->num_dpagemaps)); 406 + shrinker_free(shrinker->shrink); 407 + kfree(shrinker); 408 + } 409 + 410 + /** 411 + * drm_pagemap_shrinker_create_devm() - Create and register a pagemap shrinker 412 + * @drm: The drm device 413 + * 414 + * Create and register a pagemap shrinker that shrinks unused pagemaps 415 + * and thereby reduces memory footprint. 416 + * The shrinker is drm_device managed and unregisters itself when 417 + * the drm device is removed. 418 + * 419 + * Return: %0 on success, negative error code on failure. 420 + */ 421 + struct drm_pagemap_shrinker *drm_pagemap_shrinker_create_devm(struct drm_device *drm) 422 + { 423 + struct drm_pagemap_shrinker *shrinker; 424 + struct shrinker *shrink; 425 + int err; 426 + 427 + shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL); 428 + if (!shrinker) 429 + return ERR_PTR(-ENOMEM); 430 + 431 + shrink = shrinker_alloc(0, "drm-drm_pagemap:%s", drm->unique); 432 + if (!shrink) { 433 + kfree(shrinker); 434 + return ERR_PTR(-ENOMEM); 435 + } 436 + 437 + spin_lock_init(&shrinker->lock); 438 + INIT_LIST_HEAD(&shrinker->dpagemaps); 439 + shrinker->drm = drm; 440 + shrinker->shrink = shrink; 441 + shrink->count_objects = drm_pagemap_shrinker_count; 442 + shrink->scan_objects = drm_pagemap_shrinker_scan; 443 + shrink->private_data = shrinker; 444 + shrinker_register(shrink); 445 + 446 + err = devm_add_action_or_reset(drm->dev, drm_pagemap_shrinker_fini, shrinker); 447 + if (err) 448 + return ERR_PTR(err); 449 + 450 + return shrinker; 451 + } 452 + EXPORT_SYMBOL(drm_pagemap_shrinker_create_devm); 453 + 454 + /** 455 + * struct drm_pagemap_owner - Device interconnect group 456 + * @kref: Reference count. 457 + * 458 + * A struct drm_pagemap_owner identifies a device interconnect group. 459 + */ 460 + struct drm_pagemap_owner { 461 + struct kref kref; 462 + }; 463 + 464 + static void drm_pagemap_owner_release(struct kref *kref) 465 + { 466 + kfree(container_of(kref, struct drm_pagemap_owner, kref)); 467 + } 468 + 469 + /** 470 + * drm_pagemap_release_owner() - Stop participating in an interconnect group 471 + * @peer: Pointer to the struct drm_pagemap_peer used when joining the group 472 + * 473 + * Stop participating in an interconnect group. This function is typically 474 + * called when a pagemap is removed to indicate that it doesn't need to 475 + * be taken into account. 476 + */ 477 + void drm_pagemap_release_owner(struct drm_pagemap_peer *peer) 478 + { 479 + struct drm_pagemap_owner_list *owner_list = peer->list; 480 + 481 + if (!owner_list) 482 + return; 483 + 484 + mutex_lock(&owner_list->lock); 485 + list_del(&peer->link); 486 + kref_put(&peer->owner->kref, drm_pagemap_owner_release); 487 + peer->owner = NULL; 488 + mutex_unlock(&owner_list->lock); 489 + } 490 + EXPORT_SYMBOL(drm_pagemap_release_owner); 491 + 492 + /** 493 + * typedef interconnect_fn - Callback function to identify fast interconnects 494 + * @peer1: First endpoint. 495 + * @peer2: Second endpont. 496 + * 497 + * The function returns %true iff @peer1 and @peer2 have a fast interconnect. 498 + * Note that this is symmetrical. The function has no notion of client and provider, 499 + * which may not be sufficient in some cases. However, since the callback is intended 500 + * to guide in providing common pagemap owners, the notion of a common owner to 501 + * indicate fast interconnects would then have to change as well. 502 + * 503 + * Return: %true iff @peer1 and @peer2 have a fast interconnect. Otherwise @false. 504 + */ 505 + typedef bool (*interconnect_fn)(struct drm_pagemap_peer *peer1, struct drm_pagemap_peer *peer2); 506 + 507 + /** 508 + * drm_pagemap_acquire_owner() - Join an interconnect group 509 + * @peer: A struct drm_pagemap_peer keeping track of the device interconnect 510 + * @owner_list: Pointer to the owner_list, keeping track of all interconnects 511 + * @has_interconnect: Callback function to determine whether two peers have a 512 + * fast local interconnect. 513 + * 514 + * Repeatedly calls @has_interconnect for @peer and other peers on @owner_list to 515 + * determine a set of peers for which @peer has a fast interconnect. That set will 516 + * have common &struct drm_pagemap_owner, and upon successful return, @peer::owner 517 + * will point to that struct, holding a reference, and @peer will be registered in 518 + * @owner_list. If @peer doesn't have any fast interconnects to other @peers, a 519 + * new unique &struct drm_pagemap_owner will be allocated for it, and that 520 + * may be shared with other peers that, at a later point, are determined to have 521 + * a fast interconnect with @peer. 522 + * 523 + * When @peer no longer participates in an interconnect group, 524 + * drm_pagemap_release_owner() should be called to drop the reference on the 525 + * struct drm_pagemap_owner. 526 + * 527 + * Return: %0 on success, negative error code on failure. 528 + */ 529 + int drm_pagemap_acquire_owner(struct drm_pagemap_peer *peer, 530 + struct drm_pagemap_owner_list *owner_list, 531 + interconnect_fn has_interconnect) 532 + { 533 + struct drm_pagemap_peer *cur_peer; 534 + struct drm_pagemap_owner *owner = NULL; 535 + bool interconnect = false; 536 + 537 + mutex_lock(&owner_list->lock); 538 + might_alloc(GFP_KERNEL); 539 + list_for_each_entry(cur_peer, &owner_list->peers, link) { 540 + if (cur_peer->owner != owner) { 541 + if (owner && interconnect) 542 + break; 543 + owner = cur_peer->owner; 544 + interconnect = true; 545 + } 546 + if (interconnect && !has_interconnect(peer, cur_peer)) 547 + interconnect = false; 548 + } 549 + 550 + if (!interconnect) { 551 + owner = kmalloc(sizeof(*owner), GFP_KERNEL); 552 + if (!owner) { 553 + mutex_unlock(&owner_list->lock); 554 + return -ENOMEM; 555 + } 556 + kref_init(&owner->kref); 557 + list_add_tail(&peer->link, &owner_list->peers); 558 + } else { 559 + kref_get(&owner->kref); 560 + list_add_tail(&peer->link, &cur_peer->link); 561 + } 562 + peer->owner = owner; 563 + peer->list = owner_list; 564 + mutex_unlock(&owner_list->lock); 565 + 566 + return 0; 567 + } 568 + EXPORT_SYMBOL(drm_pagemap_acquire_owner);

+2

drivers/gpu/drm/xe/Makefile

··· 98 98 xe_page_reclaim.o \ 99 99 xe_pat.o \ 100 100 xe_pci.o \ 101 + xe_pci_rebar.o \ 101 102 xe_pcode.o \ 102 103 xe_pm.o \ 103 104 xe_preempt_fence.o \ ··· 117 116 xe_sa.o \ 118 117 xe_sched_job.o \ 119 118 xe_shrinker.o \ 119 + xe_soc_remapper.o \ 120 120 xe_step.o \ 121 121 xe_survivability_mode.o \ 122 122 xe_sync.o \

+64

drivers/gpu/drm/xe/abi/guc_klvs_abi.h

··· 8 8 9 9 #include <linux/types.h> 10 10 11 + #include "abi/guc_scheduler_abi.h" 12 + 11 13 /** 12 14 * DOC: GuC KLV 13 15 * ··· 48 46 * Refers to 32 bit architecture version as reported by the HW IP. 49 47 * This key is supported on MTL+ platforms only. 50 48 * Requires GuC ABI 1.2+. 49 + * 50 + * _`GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE` : 0x3001 51 + * Tells the driver whether scheduler groups are enabled or not. 52 + * Requires GuC ABI 1.26+ 51 53 */ 52 54 53 55 #define GUC_KLV_GLOBAL_CFG_GMD_ID_KEY 0x3000u 54 56 #define GUC_KLV_GLOBAL_CFG_GMD_ID_LEN 1u 57 + 58 + #define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY 0x3001u 59 + #define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_LEN 1u 55 60 56 61 /** 57 62 * DOC: GuC Self Config KLVs ··· 209 200 * :0: adverse events are not counted (default) 210 201 * :n: sample period in milliseconds 211 202 * 203 + * _`GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG` : 0x8004 204 + * This config allows the PF to split the engines across scheduling groups. 205 + * Each group is independently timesliced across VFs, allowing different 206 + * VFs to be active on the HW at the same time. When enabling this feature, 207 + * all engines must be assigned to a group (and only one group), or they 208 + * will be excluded from scheduling after this KLV is sent. To enable 209 + * the groups, the driver must provide a masks array with 210 + * GUC_MAX_ENGINE_CLASSES entries for each group, with each mask indicating 211 + * which logical instances of that class belong to the group. Therefore, 212 + * the length of this KLV when enabling groups is 213 + * num_groups * GUC_MAX_ENGINE_CLASSES. To disable the groups, the driver 214 + * must send the KLV without any payload (i.e. len = 0). The maximum 215 + * number of groups is 8. 216 + * 212 217 * _`GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH` : 0x8D00 213 218 * This enum is to reset utilized HW engine after VF Switch (i.e to clean 214 219 * up Stale HW register left behind by previous VF) ··· 236 213 237 214 #define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY 0x8002 238 215 #define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_LEN 1u 216 + 217 + #define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY 0x8004 218 + #define GUC_KLV_VGT_POLICY_ENGINE_GROUP_MAX_COUNT GUC_MAX_SCHED_GROUPS 219 + #define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_MIN_LEN 0 220 + #define GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_MAX_LEN \ 221 + (GUC_KLV_VGT_POLICY_ENGINE_GROUP_MAX_COUNT * GUC_MAX_ENGINE_CLASSES) 239 222 240 223 #define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY 0x8D00 241 224 #define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_LEN 1u ··· 297 268 * it to take effect. Such cases might typically happen on a 1PF+1VF 298 269 * Virtualization config enabled for heavier workloads like AI/ML. 299 270 * 271 + * If scheduling groups are supported, the provided value is applied to all 272 + * groups (even if they've not yet been enabled). Support for this feature 273 + * is available from GuC 70.53.0. 274 + * 300 275 * The max value for this KLV is 100 seconds, anything exceeding that 301 276 * will be clamped to the max. 302 277 * ··· 322 289 * the KLV to force it to take effect. Such cases might typically happen 323 290 * on a 1PF+1VF Virtualization config enabled for heavier workloads like 324 291 * AI/ML. 292 + * 293 + * If scheduling groups are supported, the provided value is applied to all 294 + * groups (even if they've not yet been enabled). Support for this feature 295 + * is available from GuC 70.53.0. 325 296 * 326 297 * The max value for this KLV is 100 seconds, anything exceeding that 327 298 * will be clamped to the max. ··· 395 358 * groups and cause the latter to be turned off when registered with the 396 359 * GuC, this config allows the PF to set a threshold for multi-LRC context 397 360 * registrations by VFs to monitor their behavior. 361 + * 362 + * _`GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM' : 0x8A0E 363 + * This config sets the VFs-execution-quantum for each scheduling group in 364 + * milliseconds. The driver must provide an array of values, with each of 365 + * them matching the respective group index (first value goes to group 0, 366 + * second to group 1, etc). The setting of group values follows the same 367 + * behavior and rules as setting via GUC_KLV_VF_CFG_EXEC_QUANTUM. Note that 368 + * the GuC always sets the EQ for all groups (even the non-enabled ones), 369 + * so if we provide fewer values than the max the GuC will use 0 for the 370 + * remaining groups. This KLV is available starting from GuC 70.53.0. 371 + * 372 + * _`GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT' : 0x8A0F 373 + * This config sets the VFs-preemption-timeout for each scheduling group in 374 + * microseconds. The driver must provide an array of values, with each of 375 + * them matching the respective group index (first value goes to group 0, 376 + * second to group 1, etc). The setting of group values follows the same 377 + * behavior and rules as setting via GUC_KLV_VF_CFG_PREEMPT_TIMEOUT. Note 378 + * that the GuC always sets the EQ for all groups (even the non-enabled 379 + * ones), so if we provide fewer values than the max the GuC will use 0 for 380 + * the remaining groups. This KLV is available starting from GuC 70.53.0. 398 381 */ 399 382 400 383 #define GUC_KLV_VF_CFG_GGTT_START_KEY 0x0001 ··· 476 419 #define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY 0x8a0d 477 420 #define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN 1u 478 421 422 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY 0x8a0e 423 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MIN_LEN 1u 424 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN GUC_MAX_SCHED_GROUPS 425 + 426 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY 0x8a0f 427 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MIN_LEN 1u 428 + #define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN GUC_MAX_SCHED_GROUPS 479 429 /* 480 430 * Workaround keys: 481 431 */

+57

drivers/gpu/drm/xe/abi/guc_scheduler_abi.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef _ABI_GUC_SCHEDULER_ABI_H 7 + #define _ABI_GUC_SCHEDULER_ABI_H 8 + 9 + #include <linux/types.h> 10 + 11 + /** 12 + * Generic defines required for registration with and submissions to the GuC 13 + * scheduler. Includes engine class/instance defines and context attributes 14 + * (id, priority, etc) 15 + */ 16 + 17 + /* Engine classes/instances */ 18 + #define GUC_RENDER_CLASS 0 19 + #define GUC_VIDEO_CLASS 1 20 + #define GUC_VIDEOENHANCE_CLASS 2 21 + #define GUC_BLITTER_CLASS 3 22 + #define GUC_COMPUTE_CLASS 4 23 + #define GUC_GSC_OTHER_CLASS 5 24 + #define GUC_LAST_ENGINE_CLASS GUC_GSC_OTHER_CLASS 25 + #define GUC_MAX_ENGINE_CLASSES 16 26 + #define GUC_MAX_INSTANCES_PER_CLASS 32 27 + 28 + /* context priority values */ 29 + #define GUC_CLIENT_PRIORITY_KMD_HIGH 0 30 + #define GUC_CLIENT_PRIORITY_HIGH 1 31 + #define GUC_CLIENT_PRIORITY_KMD_NORMAL 2 32 + #define GUC_CLIENT_PRIORITY_NORMAL 3 33 + #define GUC_CLIENT_PRIORITY_NUM 4 34 + 35 + /* Context registration */ 36 + #define GUC_ID_MAX 65535 37 + #define GUC_ID_UNKNOWN 0xffffffff 38 + 39 + #define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) 40 + #define CONTEXT_REGISTRATION_FLAG_TYPE GENMASK(2, 1) 41 + #define GUC_CONTEXT_NORMAL 0 42 + #define GUC_CONTEXT_COMPRESSION_SAVE 1 43 + #define GUC_CONTEXT_COMPRESSION_RESTORE 2 44 + #define GUC_CONTEXT_COUNT (GUC_CONTEXT_COMPRESSION_RESTORE + 1) 45 + 46 + /* context enable/disable */ 47 + #define GUC_CONTEXT_DISABLE 0 48 + #define GUC_CONTEXT_ENABLE 1 49 + 50 + /* scheduler groups */ 51 + #define GUC_MAX_SCHED_GROUPS 8 52 + 53 + struct guc_sched_group { 54 + u32 engines[GUC_MAX_ENGINE_CLASSES]; 55 + } __packed; 56 + 57 + #endif

-3

drivers/gpu/drm/xe/regs/xe_pmt.h

··· 18 18 #define BMG_TELEMETRY_BASE_OFFSET 0xE0000 19 19 #define BMG_TELEMETRY_OFFSET (SOC_BASE + BMG_TELEMETRY_BASE_OFFSET) 20 20 21 - #define SG_REMAP_INDEX1 XE_REG(SOC_BASE + 0x08) 22 - #define SG_REMAP_BITS REG_GENMASK(31, 24) 23 - 24 21 #define BMG_MODS_RESIDENCY_OFFSET (0x4D0) 25 22 #define BMG_G2_RESIDENCY_OFFSET (0x530) 26 23 #define BMG_G6_RESIDENCY_OFFSET (0x538)

+14

drivers/gpu/drm/xe/regs/xe_soc_remapper_regs.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + #ifndef _XE_SOC_REMAPPER_REGS_H_ 6 + #define _XE_SOC_REMAPPER_REGS_H_ 7 + 8 + #include "xe_regs.h" 9 + 10 + #define SG_REMAP_INDEX1 XE_REG(SOC_BASE + 0x08) 11 + #define SG_REMAP_TELEM_MASK REG_GENMASK(31, 24) 12 + #define SG_REMAP_SYSCTRL_MASK REG_GENMASK(23, 16) 13 + 14 + #endif

+25

drivers/gpu/drm/xe/xe_device.c

··· 16 16 #include <drm/drm_gem_ttm_helper.h> 17 17 #include <drm/drm_ioctl.h> 18 18 #include <drm/drm_managed.h> 19 + #include <drm/drm_pagemap_util.h> 19 20 #include <drm/drm_print.h> 20 21 #include <uapi/drm/xe_drm.h> 21 22 ··· 62 61 #include "xe_pxp.h" 63 62 #include "xe_query.h" 64 63 #include "xe_shrinker.h" 64 + #include "xe_soc_remapper.h" 65 65 #include "xe_survivability_mode.h" 66 66 #include "xe_sriov.h" 67 + #include "xe_svm.h" 67 68 #include "xe_tile.h" 68 69 #include "xe_ttm_stolen_mgr.h" 69 70 #include "xe_ttm_sys_mgr.h" ··· 379 376 .fop_flags = FOP_UNSIGNED_OFFSET, 380 377 }; 381 378 379 + /** 380 + * xe_is_xe_file() - Is the file an xe device file? 381 + * @file: The file. 382 + * 383 + * Checks whether the file is opened against 384 + * an xe device. 385 + * 386 + * Return: %true if an xe file, %false if not. 387 + */ 388 + bool xe_is_xe_file(const struct file *file) 389 + { 390 + return file->f_op == &xe_driver_fops; 391 + } 392 + 382 393 static struct drm_driver driver = { 383 394 /* Don't use MTRRs here; the Xserver or userspace app should 384 395 * deal with them for Intel hardware. ··· 488 471 init_waitqueue_head(&xe->ufence_wq); 489 472 490 473 init_rwsem(&xe->usm.lock); 474 + 475 + err = xe_pagemap_shrinker_create(xe); 476 + if (err) 477 + goto err; 491 478 492 479 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 493 480 ··· 989 968 return err; 990 969 991 970 xe_nvm_init(xe); 971 + 972 + err = xe_soc_remapper_init(xe); 973 + if (err) 974 + return err; 992 975 993 976 err = xe_heci_gsc_init(xe); 994 977 if (err)

+2

drivers/gpu/drm/xe/xe_device.h

··· 200 200 201 201 int xe_is_injection_active(void); 202 202 203 + bool xe_is_xe_file(const struct file *file); 204 + 203 205 /* 204 206 * Occasionally it is seen that the G2H worker starts running after a delay of more than 205 207 * a second even after being queued and activated by the Linux workqueue subsystem. This

+22

drivers/gpu/drm/xe/xe_device_types.h

··· 36 36 #define TEST_VM_OPS_ERROR 37 37 #endif 38 38 39 + struct dram_info; 40 + struct drm_pagemap_shrinker; 39 41 struct intel_display; 40 42 struct intel_dg_nvm_dev; 41 43 struct xe_ggtt; ··· 334 332 u8 has_pxp:1; 335 333 /** @info.has_range_tlb_inval: Has range based TLB invalidations */ 336 334 u8 has_range_tlb_inval:1; 335 + /** @info.has_soc_remapper_sysctrl: Has SoC remapper system controller */ 336 + u8 has_soc_remapper_sysctrl:1; 337 + /** @info.has_soc_remapper_telem: Has SoC remapper telemetry support */ 338 + u8 has_soc_remapper_telem:1; 337 339 /** @info.has_sriov: Supports SR-IOV */ 338 340 u8 has_sriov:1; 339 341 /** @info.has_usm: Device has unified shared memory support */ ··· 455 449 #define XE_PAGEFAULT_QUEUE_COUNT 4 456 450 /** @usm.pf_queue: Page fault queues */ 457 451 struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT]; 452 + #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 453 + /** @usm.pagemap_shrinker: Shrinker for unused pagemaps */ 454 + struct drm_pagemap_shrinker *dpagemap_shrinker; 455 + #endif 458 456 } usm; 459 457 460 458 /** @pinned: pinned BO state */ ··· 581 571 /** @pmt.lock: protect access for telemetry data */ 582 572 struct mutex lock; 583 573 } pmt; 574 + 575 + /** @soc_remapper: SoC remapper object */ 576 + struct { 577 + /** @soc_remapper.lock: Serialize access to SoC Remapper's index registers */ 578 + spinlock_t lock; 579 + 580 + /** @soc_remapper.set_telem_region: Set telemetry index */ 581 + void (*set_telem_region)(struct xe_device *xe, u32 index); 582 + 583 + /** @soc_remapper.set_sysctrl_region: Set system controller index */ 584 + void (*set_sysctrl_region)(struct xe_device *xe, u32 index); 585 + } soc_remapper; 584 586 585 587 /** 586 588 * @pm_callback_task: Track the active task that is running in either

+19

drivers/gpu/drm/xe/xe_exec_queue.c

··· 17 17 #include "xe_dep_scheduler.h" 18 18 #include "xe_device.h" 19 19 #include "xe_gt.h" 20 + #include "xe_gt_sriov_pf.h" 20 21 #include "xe_gt_sriov_vf.h" 21 22 #include "xe_hw_engine_class_sysfs.h" 22 23 #include "xe_hw_engine_group.h" ··· 1109 1108 return return_mask; 1110 1109 } 1111 1110 1111 + static bool has_sched_groups(struct xe_gt *gt) 1112 + { 1113 + if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_sriov_pf_sched_groups_enabled(gt)) 1114 + return true; 1115 + 1116 + if (IS_SRIOV_VF(gt_to_xe(gt)) && xe_gt_sriov_vf_sched_groups_enabled(gt)) 1117 + return true; 1118 + 1119 + return false; 1120 + } 1121 + 1112 1122 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 1113 1123 struct drm_file *file) 1114 1124 { ··· 1210 1198 up_read(&vm->lock); 1211 1199 xe_vm_put(vm); 1212 1200 return -ENOENT; 1201 + } 1202 + 1203 + /* SRIOV sched groups are not compatible with multi-lrc */ 1204 + if (XE_IOCTL_DBG(xe, args->width > 1 && has_sched_groups(hwe->gt))) { 1205 + up_read(&vm->lock); 1206 + xe_vm_put(vm); 1207 + return -EINVAL; 1213 1208 } 1214 1209 1215 1210 q = xe_exec_queue_create(xe, vm, logical_mask,

+2 -1

drivers/gpu/drm/xe/xe_gen_wa_oob.c

··· 195 195 } 196 196 } 197 197 198 - fprintf(args[ARGS_CHEADER].f, HEADER, args[ARGS_INPUT].fn, prefix, prefix); 198 + fprintf(args[ARGS_CHEADER].f, HEADER, xbasename(args[ARGS_INPUT].fn), 199 + prefix, prefix); 199 200 200 201 ret = parse(args[ARGS_INPUT].f, args[ARGS_CSOURCE].f, 201 202 args[ARGS_CHEADER].f, prefix);

+11 -1

drivers/gpu/drm/xe/xe_gt.h

··· 20 20 for_each_if(((hwe__) = (gt__)->hw_engines + (id__)) && \ 21 21 xe_hw_engine_is_valid((hwe__))) 22 22 23 - #define CCS_MASK(gt) (((gt)->info.engine_mask & XE_HW_ENGINE_CCS_MASK) >> XE_HW_ENGINE_CCS0) 23 + #define XE_ENGINE_INSTANCES_FROM_MASK(gt, NAME) \ 24 + (((gt)->info.engine_mask & XE_HW_ENGINE_##NAME##_MASK) >> XE_HW_ENGINE_##NAME##0) 25 + 26 + #define RCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, RCS) 27 + #define VCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, VCS) 28 + #define VECS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, VECS) 29 + #define CCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, CCS) 30 + #define GSCCS_INSTANCES(gt) XE_ENGINE_INSTANCES_FROM_MASK(gt, GSCCS) 31 + 32 + /* Our devices have up to 4 media slices */ 33 + #define MAX_MEDIA_SLICES 4 24 34 25 35 #define GT_VER(gt) ({ \ 26 36 typeof(gt) gt_ = (gt); \

+4 -4

drivers/gpu/drm/xe/xe_gt_ccs_mode.c

··· 17 17 static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines) 18 18 { 19 19 u32 mode = CCS_MODE_CSLICE_0_3_MASK; /* disable all by default */ 20 - int num_slices = hweight32(CCS_MASK(gt)); 20 + int num_slices = hweight32(CCS_INSTANCES(gt)); 21 21 struct xe_device *xe = gt_to_xe(gt); 22 22 int width, cslice = 0; 23 23 u32 config = 0; ··· 59 59 config |= BIT(hwe->instance) << XE_HW_ENGINE_CCS0; 60 60 61 61 /* If a slice is fused off, leave disabled */ 62 - while ((CCS_MASK(gt) & BIT(cslice)) == 0) 62 + while ((CCS_INSTANCES(gt) & BIT(cslice)) == 0) 63 63 cslice++; 64 64 65 65 mode &= ~CCS_MODE_CSLICE(cslice, CCS_MODE_CSLICE_MASK); ··· 94 94 { 95 95 struct xe_gt *gt = kobj_to_gt(&kdev->kobj); 96 96 97 - return sysfs_emit(buf, "%u\n", hweight32(CCS_MASK(gt))); 97 + return sysfs_emit(buf, "%u\n", hweight32(CCS_INSTANCES(gt))); 98 98 } 99 99 100 100 static DEVICE_ATTR_RO(num_cslices); ··· 131 131 * Ensure number of engines specified is valid and there is an 132 132 * exact multiple of engines for slices. 133 133 */ 134 - num_slices = hweight32(CCS_MASK(gt)); 134 + num_slices = hweight32(CCS_INSTANCES(gt)); 135 135 if (!num_engines || num_engines > num_slices || num_slices % num_engines) { 136 136 xe_gt_dbg(gt, "Invalid compute config, %d engines %d slices\n", 137 137 num_engines, num_slices);

+1 -1

drivers/gpu/drm/xe/xe_gt_ccs_mode.h

··· 17 17 static inline bool xe_gt_ccs_mode_enabled(const struct xe_gt *gt) 18 18 { 19 19 /* Check if there are more than one compute engines available */ 20 - return hweight32(CCS_MASK(gt)) > 1; 20 + return hweight32(CCS_INSTANCES(gt)) > 1; 21 21 } 22 22 23 23 #endif

+20

drivers/gpu/drm/xe/xe_gt_sriov_pf.c

··· 14 14 #include "xe_gt_sriov_pf_control.h" 15 15 #include "xe_gt_sriov_pf_helpers.h" 16 16 #include "xe_gt_sriov_pf_migration.h" 17 + #include "xe_gt_sriov_pf_policy.h" 17 18 #include "xe_gt_sriov_pf_service.h" 18 19 #include "xe_gt_sriov_printk.h" 19 20 #include "xe_guc_submit.h" ··· 123 122 err = xe_gt_sriov_pf_config_init(gt); 124 123 if (err) 125 124 return err; 125 + 126 + xe_gt_sriov_pf_policy_init(gt); 126 127 127 128 err = xe_gt_sriov_pf_migration_init(gt); 128 129 if (err) ··· 284 281 pf_flush_restart(gt); 285 282 return 0; 286 283 } 284 + 285 + /** 286 + * xe_gt_sriov_pf_sched_groups_enabled - Check if multiple scheduler groups are 287 + * enabled 288 + * @gt: the &xe_gt 289 + * 290 + * This function is for PF use only. 291 + * 292 + * Return: true if shed groups were enabled, false otherwise. 293 + */ 294 + bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt) 295 + { 296 + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 297 + 298 + return xe_gt_sriov_pf_policy_sched_groups_enabled(gt); 299 + } 300 +

+8

drivers/gpu/drm/xe/xe_gt_sriov_pf.h

··· 6 6 #ifndef _XE_GT_SRIOV_PF_H_ 7 7 #define _XE_GT_SRIOV_PF_H_ 8 8 9 + #include <linux/types.h> 10 + 9 11 struct xe_gt; 10 12 11 13 #ifdef CONFIG_PCI_IOV ··· 18 16 void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid); 19 17 void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt); 20 18 void xe_gt_sriov_pf_restart(struct xe_gt *gt); 19 + bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt); 21 20 #else 22 21 static inline int xe_gt_sriov_pf_init_early(struct xe_gt *gt) 23 22 { ··· 40 37 41 38 static inline void xe_gt_sriov_pf_restart(struct xe_gt *gt) 42 39 { 40 + } 41 + 42 + static inline bool xe_gt_sriov_pf_sched_groups_enabled(struct xe_gt *gt) 43 + { 44 + return false; 43 45 } 44 46 #endif 45 47

+268 -11

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c

··· 195 195 return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs)); 196 196 } 197 197 198 + static int pf_push_vf_grp_cfg_u32(struct xe_gt *gt, unsigned int vfid, 199 + u16 key, const u32 *values, u32 count) 200 + { 201 + CLASS(xe_guc_buf, buf)(&gt->uc.guc.buf, GUC_KLV_LEN_MIN + GUC_MAX_SCHED_GROUPS); 202 + u32 *klv; 203 + 204 + xe_gt_assert(gt, count && count <= GUC_MAX_SCHED_GROUPS); 205 + 206 + if (!xe_guc_buf_is_valid(buf)) 207 + return -ENOBUFS; 208 + 209 + klv = xe_guc_buf_cpu_ptr(buf); 210 + 211 + klv[0] = FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, count); 212 + memcpy(&klv[1], values, count * sizeof(u32)); 213 + 214 + return pf_push_vf_buf_klvs(gt, vfid, 1, buf, GUC_KLV_LEN_MIN + count); 215 + } 216 + 198 217 static int pf_push_vf_cfg_exec_quantum(struct xe_gt *gt, unsigned int vfid, u32 *exec_quantum) 199 218 { 200 219 /* GuC will silently clamp values exceeding max */ ··· 287 268 return encode_ggtt(cfg, node->base.start, node->base.size, details); 288 269 } 289 270 271 + static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, 272 + const struct xe_gt_sriov_config *config) 273 + { 274 + int i; 275 + 276 + if (xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) { 277 + BUILD_BUG_ON(ARRAY_SIZE(config->exec_quantum) > 278 + GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN); 279 + BUILD_BUG_ON(ARRAY_SIZE(config->preempt_timeout) > 280 + GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN); 281 + 282 + cfg[n++] = PREP_GUC_KLV_CONST(GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY, 283 + ARRAY_SIZE(config->exec_quantum)); 284 + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) 285 + cfg[n++] = config->exec_quantum[i]; 286 + 287 + cfg[n++] = PREP_GUC_KLV_CONST(GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY, 288 + ARRAY_SIZE(config->preempt_timeout)); 289 + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) 290 + cfg[n++] = config->preempt_timeout[i]; 291 + } else { 292 + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_EXEC_QUANTUM); 293 + cfg[n++] = config->exec_quantum[0]; 294 + 295 + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT); 296 + cfg[n++] = config->preempt_timeout[0]; 297 + } 298 + 299 + return n; 300 + } 301 + 290 302 /* Return: number of configuration dwords written */ 291 303 static u32 encode_config(struct xe_gt *gt, u32 *cfg, const struct xe_gt_sriov_config *config, 292 304 bool details) ··· 348 298 cfg[n++] = upper_32_bits(xe_bo_size(config->lmem_obj)); 349 299 } 350 300 351 - cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_EXEC_QUANTUM); 352 - cfg[n++] = config->exec_quantum; 353 - 354 - cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT); 355 - cfg[n++] = config->preempt_timeout; 301 + n = encode_config_sched(gt, cfg, n, config); 356 302 357 303 #define encode_threshold_config(TAG, NAME, VER...) ({ \ 358 304 if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER)) { \ ··· 1020 974 xe_gt_sriov_info(gt, "%s provisioned with %u%s %s\n", 1021 975 name, actual, unit(actual), what); 1022 976 return 0; 977 + } 978 + 979 + static char *to_group_name(const char *what, u8 group, char *buf, size_t size) 980 + { 981 + snprintf(buf, size, "group%u%s%s", group, what ? " " : "", what ?: ""); 982 + return buf; 983 + } 984 + 985 + static int 986 + pf_groups_cfg_set_u32_done(struct xe_gt *gt, unsigned int vfid, u32 *values, u32 count, 987 + void (*get_actual)(struct xe_gt *, unsigned int, u32 *, u32), 988 + const char *what, const char *(*unit)(u32), int err) 989 + { 990 + u32 actual[GUC_MAX_SCHED_GROUPS]; 991 + char group_name[32]; 992 + u8 g; 993 + 994 + xe_gt_assert(gt, count <= ARRAY_SIZE(actual)); 995 + 996 + get_actual(gt, vfid, actual, count); 997 + 998 + for (g = 0; g < count; g++) 999 + pf_config_set_u32_done(gt, vfid, values[g], actual[g], 1000 + to_group_name(what, g, group_name, sizeof(group_name)), 1001 + unit, err); 1002 + 1003 + return err; 1023 1004 } 1024 1005 1025 1006 /** ··· 1933 1860 { 1934 1861 struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 1935 1862 int err; 1863 + int i; 1936 1864 1937 1865 err = pf_push_vf_cfg_exec_quantum(gt, vfid, &exec_quantum); 1938 1866 if (unlikely(err)) 1939 1867 return err; 1940 1868 1941 - config->exec_quantum = exec_quantum; 1869 + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) 1870 + config->exec_quantum[i] = exec_quantum; 1871 + 1942 1872 return 0; 1943 1873 } 1944 1874 ··· 1949 1873 { 1950 1874 struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 1951 1875 1952 - return config->exec_quantum; 1876 + return config->exec_quantum[0]; 1953 1877 } 1954 1878 1955 1879 /** ··· 2056 1980 exec_quantum_unit, n, err); 2057 1981 } 2058 1982 1983 + static int pf_provision_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 1984 + const u32 *exec_quantums, u32 count) 1985 + { 1986 + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 1987 + int err; 1988 + int i; 1989 + 1990 + err = pf_push_vf_grp_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY, 1991 + exec_quantums, count); 1992 + if (unlikely(err)) 1993 + return err; 1994 + 1995 + /* 1996 + * GuC silently clamps values exceeding the max and zeroes out the 1997 + * quantum for groups not in the klv payload 1998 + */ 1999 + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) { 2000 + if (i < count) 2001 + config->exec_quantum[i] = min_t(u32, exec_quantums[i], 2002 + GUC_KLV_VF_CFG_EXEC_QUANTUM_MAX_VALUE); 2003 + else 2004 + config->exec_quantum[i] = 0; 2005 + } 2006 + 2007 + return 0; 2008 + } 2009 + 2010 + static void pf_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 2011 + u32 *exec_quantums, u32 max_count) 2012 + { 2013 + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 2014 + u32 count = min_t(u32, max_count, ARRAY_SIZE(config->exec_quantum)); 2015 + 2016 + memcpy(exec_quantums, config->exec_quantum, sizeof(u32) * count); 2017 + } 2018 + 2019 + /** 2020 + * xe_gt_sriov_pf_config_set_groups_exec_quantums() - Configure PF/VF EQs for sched groups. 2021 + * @gt: the &xe_gt 2022 + * @vfid: the PF or VF identifier 2023 + * @exec_quantums: array of requested EQs in milliseconds (0 is infinity) 2024 + * @count: number of entries in the array 2025 + * 2026 + * This function can only be called on PF. 2027 + * It will log the provisioned value or an error in case of the failure. 2028 + * 2029 + * Return: 0 on success or a negative error code on failure. 2030 + */ 2031 + int xe_gt_sriov_pf_config_set_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 2032 + u32 *exec_quantums, u32 count) 2033 + { 2034 + int err; 2035 + 2036 + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); 2037 + 2038 + err = pf_provision_groups_exec_quantums(gt, vfid, exec_quantums, count); 2039 + 2040 + return pf_groups_cfg_set_u32_done(gt, vfid, exec_quantums, count, 2041 + pf_get_groups_exec_quantums, 2042 + "execution quantum", 2043 + exec_quantum_unit, err); 2044 + } 2045 + 2046 + /** 2047 + * xe_gt_sriov_pf_config_get_groups_exec_quantums() - Get PF/VF sched groups EQs 2048 + * @gt: the &xe_gt 2049 + * @vfid: the PF or VF identifier 2050 + * @exec_quantums: array in which to store the execution quantums values 2051 + * @count: maximum number of entries to store 2052 + * 2053 + * This function can only be called on PF. 2054 + */ 2055 + void xe_gt_sriov_pf_config_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 2056 + u32 *exec_quantums, u32 count) 2057 + { 2058 + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); 2059 + 2060 + xe_gt_assert(gt, count <= GUC_MAX_SCHED_GROUPS); 2061 + 2062 + pf_get_groups_exec_quantums(gt, vfid, exec_quantums, count); 2063 + } 2064 + 2059 2065 static const char *preempt_timeout_unit(u32 preempt_timeout) 2060 2066 { 2061 2067 return preempt_timeout ? "us" : "(infinity)"; ··· 2148 1990 { 2149 1991 struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 2150 1992 int err; 1993 + int i; 2151 1994 2152 1995 err = pf_push_vf_cfg_preempt_timeout(gt, vfid, &preempt_timeout); 2153 1996 if (unlikely(err)) 2154 1997 return err; 2155 1998 2156 - config->preempt_timeout = preempt_timeout; 1999 + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) 2000 + config->preempt_timeout[i] = preempt_timeout; 2157 2001 2158 2002 return 0; 2159 2003 } ··· 2164 2004 { 2165 2005 struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 2166 2006 2167 - return config->preempt_timeout; 2007 + return config->preempt_timeout[0]; 2168 2008 } 2169 2009 2170 2010 /** ··· 2270 2110 preempt_timeout_unit, n, err); 2271 2111 } 2272 2112 2113 + static int pf_provision_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 2114 + const u32 *preempt_timeouts, u32 count) 2115 + { 2116 + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 2117 + int err; 2118 + int i; 2119 + 2120 + err = pf_push_vf_grp_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY, 2121 + preempt_timeouts, count); 2122 + if (unlikely(err)) 2123 + return err; 2124 + 2125 + /* 2126 + * GuC silently clamps values exceeding the max and zeroes out the 2127 + * quantum for groups not in the klv payload 2128 + */ 2129 + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) { 2130 + if (i < count) 2131 + config->preempt_timeout[i] = 2132 + min_t(u32, preempt_timeouts[i], 2133 + GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_MAX_VALUE); 2134 + else 2135 + config->preempt_timeout[i] = 0; 2136 + } 2137 + 2138 + return 0; 2139 + } 2140 + 2141 + static void pf_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 2142 + u32 *preempt_timeouts, u32 max_count) 2143 + { 2144 + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); 2145 + u32 count = min_t(u32, max_count, ARRAY_SIZE(config->preempt_timeout)); 2146 + 2147 + memcpy(preempt_timeouts, config->preempt_timeout, sizeof(u32) * count); 2148 + } 2149 + 2150 + /** 2151 + * xe_gt_sriov_pf_config_set_groups_preempt_timeouts() - Configure PF/VF PTs for sched groups. 2152 + * @gt: the &xe_gt 2153 + * @vfid: the PF or VF identifier 2154 + * @preempt_timeouts: array of requested PTs in microseconds (0 is infinity) 2155 + * @count: number of entries in the array 2156 + * 2157 + * This function can only be called on PF. 2158 + * It will log the provisioned value or an error in case of the failure. 2159 + * 2160 + * Return: 0 on success or a negative error code on failure. 2161 + */ 2162 + int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 2163 + u32 *preempt_timeouts, u32 count) 2164 + { 2165 + int err; 2166 + 2167 + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); 2168 + 2169 + err = pf_provision_groups_preempt_timeouts(gt, vfid, preempt_timeouts, count); 2170 + 2171 + return pf_groups_cfg_set_u32_done(gt, vfid, preempt_timeouts, count, 2172 + pf_get_groups_preempt_timeouts, 2173 + "preempt_timeout", 2174 + preempt_timeout_unit, err); 2175 + } 2176 + 2177 + /** 2178 + * xe_gt_sriov_pf_config_get_groups_preempt_timeouts() - Get PF/VF sched groups PTs 2179 + * @gt: the &xe_gt 2180 + * @vfid: the PF or VF identifier 2181 + * @preempt_timeouts: array in which to store the preemption timeouts values 2182 + * @count: maximum number of entries to store 2183 + * 2184 + * This function can only be called on PF. 2185 + */ 2186 + void xe_gt_sriov_pf_config_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 2187 + u32 *preempt_timeouts, u32 count) 2188 + { 2189 + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); 2190 + 2191 + xe_gt_assert(gt, count <= GUC_MAX_SCHED_GROUPS); 2192 + 2193 + pf_get_groups_preempt_timeouts(gt, vfid, preempt_timeouts, count); 2194 + } 2195 + 2273 2196 static const char *sched_priority_unit(u32 priority) 2274 2197 { 2275 2198 return priority == GUC_SCHED_PRIORITY_LOW ? "(low)" : ··· 2426 2183 2427 2184 static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config) 2428 2185 { 2186 + int i; 2187 + 2429 2188 lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); 2430 2189 2431 - config->exec_quantum = 0; 2432 - config->preempt_timeout = 0; 2190 + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) { 2191 + config->exec_quantum[i] = 0; 2192 + config->preempt_timeout[i] = 0; 2193 + } 2433 2194 } 2434 2195 2435 2196 static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid, ··· 2794 2547 if (len != GUC_KLV_VF_CFG_EXEC_QUANTUM_LEN) 2795 2548 return -EBADMSG; 2796 2549 return pf_provision_exec_quantum(gt, vfid, value[0]); 2550 + 2551 + case GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY: 2552 + if (len > GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_MAX_LEN) 2553 + return -EBADMSG; 2554 + return pf_provision_groups_exec_quantums(gt, vfid, value, len); 2555 + 2556 + case GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY: 2557 + if (len > GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN) 2558 + return -EBADMSG; 2559 + return pf_provision_groups_preempt_timeouts(gt, vfid, value, len); 2797 2560 2798 2561 case GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY: 2799 2562 if (len != GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_LEN)

+10

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h

··· 46 46 u32 exec_quantum); 47 47 int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum); 48 48 49 + void xe_gt_sriov_pf_config_get_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 50 + u32 *exec_quantum, u32 max_count); 51 + int xe_gt_sriov_pf_config_set_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, 52 + u32 *exec_quantum, u32 count); 53 + 49 54 u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid); 50 55 int xe_gt_sriov_pf_config_set_preempt_timeout(struct xe_gt *gt, unsigned int vfid, 51 56 u32 preempt_timeout); ··· 59 54 int xe_gt_sriov_pf_config_set_preempt_timeout_locked(struct xe_gt *gt, unsigned int vfid, 60 55 u32 preempt_timeout); 61 56 int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout); 57 + 58 + void xe_gt_sriov_pf_config_get_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 59 + u32 *preempt_timeout, u32 max_count); 60 + int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, 61 + u32 *preempt_timeout, u32 count); 62 62 63 63 u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid); 64 64 int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority);

+3 -2

drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h

··· 6 6 #ifndef _XE_GT_SRIOV_PF_CONFIG_TYPES_H_ 7 7 #define _XE_GT_SRIOV_PF_CONFIG_TYPES_H_ 8 8 9 + #include "abi/guc_scheduler_abi.h" 9 10 #include "xe_ggtt_types.h" 10 11 #include "xe_guc_klv_thresholds_set_types.h" 11 12 ··· 31 30 /** @begin_db: start index of GuC doorbell ID range. */ 32 31 u16 begin_db; 33 32 /** @exec_quantum: execution-quantum in milliseconds. */ 34 - u32 exec_quantum; 33 + u32 exec_quantum[GUC_MAX_SCHED_GROUPS]; 35 34 /** @preempt_timeout: preemption timeout in microseconds. */ 36 - u32 preempt_timeout; 35 + u32 preempt_timeout[GUC_MAX_SCHED_GROUPS]; 37 36 /** @sched_priority: scheduling priority. */ 38 37 u32 sched_priority; 39 38 /** @thresholds: GuC thresholds for adverse events notifications. */

+295

drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c

··· 160 160 * /sys/kernel/debug/dri/BDF/ 161 161 * ├── sriov 162 162 * : ├── pf 163 + * : ├── tile0 164 + * : ├── gt0 165 + * : ├── sched_groups_mode 166 + * ├── sched_groups_exec_quantums_ms 167 + * ├── sched_groups_preempt_timeout_us 168 + * ├── sched_groups 169 + * : ├── group0 170 + * : 171 + * : └── groupN 172 + * ├── vf1 173 + * : ├── tile0 174 + * : ├── gt0 175 + * : ├── sched_groups_exec_quantums_ms 176 + * ├── sched_groups_preempt_timeout_us 177 + * : 178 + */ 179 + 180 + static const char *sched_group_mode_to_string(enum xe_sriov_sched_group_modes mode) 181 + { 182 + switch (mode) { 183 + case XE_SRIOV_SCHED_GROUPS_DISABLED: 184 + return "disabled"; 185 + case XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: 186 + return "media_slices"; 187 + case XE_SRIOV_SCHED_GROUPS_MODES_COUNT: 188 + /* dummy mode to make the compiler happy */ 189 + break; 190 + } 191 + 192 + return "unknown"; 193 + } 194 + 195 + static int sched_groups_info(struct seq_file *m, void *data) 196 + { 197 + struct drm_printer p = drm_seq_file_printer(m); 198 + struct xe_gt *gt = extract_gt(m->private); 199 + enum xe_sriov_sched_group_modes current_mode = 200 + gt->sriov.pf.policy.guc.sched_groups.current_mode; 201 + enum xe_sriov_sched_group_modes mode; 202 + 203 + for (mode = XE_SRIOV_SCHED_GROUPS_DISABLED; 204 + mode < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; 205 + mode++) { 206 + if (!xe_sriov_gt_pf_policy_has_sched_group_mode(gt, mode)) 207 + continue; 208 + 209 + drm_printf(&p, "%s%s%s%s", 210 + mode == XE_SRIOV_SCHED_GROUPS_DISABLED ? "" : " ", 211 + mode == current_mode ? "[" : "", 212 + sched_group_mode_to_string(mode), 213 + mode == current_mode ? "]" : ""); 214 + } 215 + 216 + drm_puts(&p, "\n"); 217 + 218 + return 0; 219 + } 220 + 221 + static int sched_groups_open(struct inode *inode, struct file *file) 222 + { 223 + return single_open(file, sched_groups_info, inode->i_private); 224 + } 225 + 226 + static ssize_t sched_groups_write(struct file *file, const char __user *ubuf, 227 + size_t size, loff_t *pos) 228 + { 229 + struct xe_gt *gt = extract_gt(file_inode(file)->i_private); 230 + enum xe_sriov_sched_group_modes mode; 231 + char name[32]; 232 + int ret; 233 + 234 + if (*pos) 235 + return -ESPIPE; 236 + 237 + if (!size) 238 + return -ENODATA; 239 + 240 + if (size > sizeof(name) - 1) 241 + return -EINVAL; 242 + 243 + ret = simple_write_to_buffer(name, sizeof(name) - 1, pos, ubuf, size); 244 + if (ret < 0) 245 + return ret; 246 + name[ret] = '\0'; 247 + 248 + for (mode = XE_SRIOV_SCHED_GROUPS_DISABLED; 249 + mode < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; 250 + mode++) 251 + if (sysfs_streq(name, sched_group_mode_to_string(mode))) 252 + break; 253 + 254 + if (mode == XE_SRIOV_SCHED_GROUPS_MODES_COUNT) 255 + return -EINVAL; 256 + 257 + guard(xe_pm_runtime)(gt_to_xe(gt)); 258 + ret = xe_gt_sriov_pf_policy_set_sched_groups_mode(gt, mode); 259 + 260 + return ret < 0 ? ret : size; 261 + } 262 + 263 + static const struct file_operations sched_groups_fops = { 264 + .owner = THIS_MODULE, 265 + .open = sched_groups_open, 266 + .read = seq_read, 267 + .write = sched_groups_write, 268 + .llseek = seq_lseek, 269 + .release = single_release, 270 + }; 271 + 272 + static int sched_groups_config_show(struct seq_file *m, void *data, 273 + void (*get)(struct xe_gt *, unsigned int, u32 *, u32)) 274 + { 275 + struct drm_printer p = drm_seq_file_printer(m); 276 + unsigned int vfid = extract_vfid(m->private); 277 + struct xe_gt *gt = extract_gt(m->private); 278 + u32 values[GUC_MAX_SCHED_GROUPS]; 279 + bool first = true; 280 + u8 group; 281 + 282 + get(gt, vfid, values, ARRAY_SIZE(values)); 283 + 284 + for (group = 0; group < ARRAY_SIZE(values); group++) { 285 + drm_printf(&p, "%s%u", first ? "" : ",", values[group]); 286 + 287 + first = false; 288 + } 289 + 290 + drm_puts(&p, "\n"); 291 + 292 + return 0; 293 + } 294 + 295 + static ssize_t sched_groups_config_write(struct file *file, const char __user *ubuf, 296 + size_t size, loff_t *pos, 297 + int (*set)(struct xe_gt *, unsigned int, u32 *, u32)) 298 + { 299 + struct dentry *parent = file_inode(file)->i_private; 300 + unsigned int vfid = extract_vfid(parent); 301 + struct xe_gt *gt = extract_gt(parent); 302 + u32 values[GUC_MAX_SCHED_GROUPS]; 303 + int *input __free(kfree) = NULL; 304 + u32 count; 305 + int ret; 306 + int i; 307 + 308 + if (*pos) 309 + return -ESPIPE; 310 + 311 + if (!size) 312 + return -ENODATA; 313 + 314 + ret = parse_int_array_user(ubuf, min(size, GUC_MAX_SCHED_GROUPS * sizeof(u32)), &input); 315 + if (ret) 316 + return ret; 317 + 318 + count = input[0]; 319 + if (count > GUC_MAX_SCHED_GROUPS) 320 + return -E2BIG; 321 + 322 + for (i = 0; i < count; i++) { 323 + if (input[i + 1] < 0 || input[i + 1] > S32_MAX) 324 + return -EINVAL; 325 + 326 + values[i] = input[i + 1]; 327 + } 328 + 329 + guard(xe_pm_runtime)(gt_to_xe(gt)); 330 + ret = set(gt, vfid, values, count); 331 + 332 + return ret < 0 ? ret : size; 333 + } 334 + 335 + #define DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(CONFIG) \ 336 + static int sched_groups_##CONFIG##_show(struct seq_file *m, void *data) \ 337 + { \ 338 + return sched_groups_config_show(m, data, \ 339 + xe_gt_sriov_pf_config_get_groups_##CONFIG); \ 340 + } \ 341 + \ 342 + static int sched_groups_##CONFIG##_open(struct inode *inode, struct file *file) \ 343 + { \ 344 + return single_open(file, sched_groups_##CONFIG##_show, \ 345 + inode->i_private); \ 346 + } \ 347 + \ 348 + static ssize_t sched_groups_##CONFIG##_write(struct file *file, \ 349 + const char __user *ubuf, \ 350 + size_t size, loff_t *pos) \ 351 + { \ 352 + return sched_groups_config_write(file, ubuf, size, pos, \ 353 + xe_gt_sriov_pf_config_set_groups_##CONFIG); \ 354 + } \ 355 + \ 356 + static const struct file_operations sched_groups_##CONFIG##_fops = { \ 357 + .owner = THIS_MODULE, \ 358 + .open = sched_groups_##CONFIG##_open, \ 359 + .read = seq_read, \ 360 + .llseek = seq_lseek, \ 361 + .write = sched_groups_##CONFIG##_write, \ 362 + .release = single_release, \ 363 + } 364 + 365 + DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(exec_quantums); 366 + DEFINE_SRIOV_GT_GRP_CFG_DEBUGFS_ATTRIBUTE(preempt_timeouts); 367 + 368 + static ssize_t sched_group_engines_read(struct file *file, char __user *buf, 369 + size_t count, loff_t *ppos) 370 + { 371 + struct dentry *dent = file_dentry(file); 372 + struct xe_gt *gt = extract_gt(dent->d_parent->d_parent); 373 + struct xe_gt_sriov_scheduler_groups *info = &gt->sriov.pf.policy.guc.sched_groups; 374 + struct guc_sched_group *groups = info->modes[info->current_mode].groups; 375 + u32 num_groups = info->modes[info->current_mode].num_groups; 376 + unsigned int group = (uintptr_t)extract_priv(dent); 377 + struct xe_hw_engine *hwe; 378 + enum xe_hw_engine_id id; 379 + char engines[128]; 380 + 381 + engines[0] = '\0'; 382 + 383 + if (group < num_groups) { 384 + for_each_hw_engine(hwe, gt, id) { 385 + u8 guc_class = xe_engine_class_to_guc_class(hwe->class); 386 + u32 mask = groups[group].engines[guc_class]; 387 + 388 + if (mask & BIT(hwe->logical_instance)) { 389 + strlcat(engines, hwe->name, sizeof(engines)); 390 + strlcat(engines, " ", sizeof(engines)); 391 + } 392 + } 393 + strlcat(engines, "\n", sizeof(engines)); 394 + } 395 + 396 + return simple_read_from_buffer(buf, count, ppos, engines, strlen(engines)); 397 + } 398 + 399 + static const struct file_operations sched_group_engines_fops = { 400 + .owner = THIS_MODULE, 401 + .open = simple_open, 402 + .read = sched_group_engines_read, 403 + .llseek = default_llseek, 404 + }; 405 + 406 + static void pf_add_sched_groups(struct xe_gt *gt, struct dentry *parent, unsigned int vfid) 407 + { 408 + struct dentry *groups; 409 + u8 group; 410 + 411 + xe_gt_assert(gt, gt == extract_gt(parent)); 412 + xe_gt_assert(gt, vfid == extract_vfid(parent)); 413 + 414 + /* 415 + * TODO: we currently call this function before we initialize scheduler 416 + * groups, so at this point in time we don't know if there are any 417 + * valid groups on the GT and we can't selectively register the debugfs 418 + * only if there are any. Therefore, we always register the debugfs 419 + * files if we're on a platform that has support for groups. 420 + * We should rework the flow so that debugfs is registered after the 421 + * policy init, so that we check if there are valid groups before 422 + * adding the debugfs files. 423 + * Similarly, instead of using GUC_MAX_SCHED_GROUPS we could use 424 + * gt->sriov.pf.policy.guc.sched_groups.max_number_of_groups. 425 + */ 426 + if (!xe_sriov_gt_pf_policy_has_sched_groups_support(gt)) 427 + return; 428 + 429 + debugfs_create_file("sched_groups_exec_quantums_ms", 0644, parent, parent, 430 + &sched_groups_exec_quantums_fops); 431 + debugfs_create_file("sched_groups_preempt_timeouts_us", 0644, parent, parent, 432 + &sched_groups_preempt_timeouts_fops); 433 + 434 + if (vfid != PFID) 435 + return; 436 + 437 + debugfs_create_file("sched_groups_mode", 0644, parent, parent, &sched_groups_fops); 438 + 439 + groups = debugfs_create_dir("sched_groups", parent); 440 + if (IS_ERR(groups)) 441 + return; 442 + 443 + for (group = 0; group < GUC_MAX_SCHED_GROUPS; group++) { 444 + char name[10]; 445 + 446 + snprintf(name, sizeof(name), "group%u", group); 447 + debugfs_create_file(name, 0644, groups, (void *)(uintptr_t)group, 448 + &sched_group_engines_fops); 449 + } 450 + } 451 + 452 + /* 453 + * /sys/kernel/debug/dri/BDF/ 454 + * ├── sriov 455 + * : ├── pf 163 456 * │ ├── tile0 164 457 * │ : ├── gt0 165 458 * │ : ├── doorbells_spare ··· 811 518 812 519 if (vfid) { 813 520 pf_add_config_attrs(gt, dent, vfid); 521 + pf_add_sched_groups(gt, dent, vfid); 814 522 815 523 debugfs_create_file("control", 0600, dent, NULL, &control_ops); 816 524 ··· 825 531 } else { 826 532 pf_add_config_attrs(gt, dent, PFID); 827 533 pf_add_policy_attrs(gt, dent); 534 + pf_add_sched_groups(gt, dent, PFID); 828 535 829 536 drm_debugfs_create_files(pf_info, ARRAY_SIZE(pf_info), dent, minor); 830 537 }

+329

drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c

··· 3 3 * Copyright © 2023-2024 Intel Corporation 4 4 */ 5 5 6 + #include <drm/drm_managed.h> 7 + 6 8 #include "abi/guc_actions_sriov_abi.h" 7 9 8 10 #include "xe_bo.h" ··· 12 10 #include "xe_gt_sriov_pf_helpers.h" 13 11 #include "xe_gt_sriov_pf_policy.h" 14 12 #include "xe_gt_sriov_printk.h" 13 + #include "xe_guc.h" 15 14 #include "xe_guc_buf.h" 16 15 #include "xe_guc_ct.h" 17 16 #include "xe_guc_klv_helpers.h" 17 + #include "xe_guc_submit.h" 18 18 #include "xe_pm.h" 19 19 20 20 /* ··· 96 92 }; 97 93 98 94 return pf_push_policy_klvs(gt, 1, klv, ARRAY_SIZE(klv)); 95 + } 96 + 97 + static int pf_push_policy_payload(struct xe_gt *gt, u16 key, void *payload, u32 num_dwords) 98 + { 99 + CLASS(xe_guc_buf, buf)(&gt->uc.guc.buf, GUC_KLV_LEN_MIN + num_dwords); 100 + u32 *klv; 101 + 102 + if (!xe_guc_buf_is_valid(buf)) 103 + return -ENOBUFS; 104 + 105 + klv = xe_guc_buf_cpu_ptr(buf); 106 + 107 + klv[0] = PREP_GUC_KLV(key, num_dwords); 108 + if (num_dwords) 109 + memcpy(&klv[1], payload, num_dwords * sizeof(u32)); 110 + 111 + return pf_push_policy_buf_klvs(gt, 1, buf, GUC_KLV_LEN_MIN + num_dwords); 99 112 } 100 113 101 114 static int pf_update_policy_bool(struct xe_gt *gt, u16 key, bool *policy, bool value) ··· 372 351 return value; 373 352 } 374 353 354 + static void pf_sched_group_media_slices(struct xe_gt *gt, struct guc_sched_group **groups, 355 + u32 *num_groups) 356 + { 357 + u8 slice_to_group[MAX_MEDIA_SLICES]; 358 + u32 vecs_mask = VECS_INSTANCES(gt); 359 + u32 gsc_mask = GSCCS_INSTANCES(gt); 360 + u32 vcs_mask = VCS_INSTANCES(gt); 361 + struct guc_sched_group *values; 362 + struct xe_hw_engine *hwe; 363 + enum xe_hw_engine_id id; 364 + int group = 0; 365 + int slice; 366 + 367 + xe_gt_assert(gt, xe_gt_is_media_type(gt)); 368 + 369 + /* 370 + * Post-BMG the matching of video engines to slices changes, so for now 371 + * we don't allow this mode on those platforms. 372 + */ 373 + if (gt_to_xe(gt)->info.platform > XE_BATTLEMAGE) 374 + return; 375 + 376 + /* 377 + * On BMG and older platforms a media slice has 2 VCS and a VECS. We 378 + * bundle the GSC with the first slice. 379 + */ 380 + for (slice = 0; slice < MAX_MEDIA_SLICES; slice++) { 381 + if ((vcs_mask & 0x3) || (vecs_mask & 0x1) || (gsc_mask & 0x1)) 382 + slice_to_group[slice] = group++; 383 + 384 + vcs_mask >>= 2; 385 + vecs_mask >>= 1; 386 + gsc_mask >>= 1; 387 + } 388 + 389 + xe_gt_assert(gt, !vcs_mask); 390 + xe_gt_assert(gt, !vecs_mask); 391 + xe_gt_assert(gt, !gsc_mask); 392 + 393 + /* We need at least 2 slices to split them up */ 394 + if (group < 2) 395 + return; 396 + 397 + /* 398 + * If we have more groups than the GuC can support then we don't want to 399 + * expose this specific mode, because the GuC will return an error if we 400 + * try to enable it. 401 + */ 402 + if (group > gt->sriov.pf.policy.guc.sched_groups.max_groups) { 403 + xe_gt_sriov_notice(gt, "media_slice mode has too many groups: %u vs %u\n", 404 + group, gt->sriov.pf.policy.guc.sched_groups.max_groups); 405 + return; 406 + } 407 + 408 + /* The GuC expects an array with a guc_sched_group entry for each group */ 409 + values = drmm_kcalloc(&gt_to_xe(gt)->drm, group, sizeof(struct guc_sched_group), 410 + GFP_KERNEL); 411 + if (!values) 412 + return; 413 + 414 + for_each_hw_engine(hwe, gt, id) { 415 + u8 guc_class = xe_engine_class_to_guc_class(hwe->class); 416 + 417 + switch (hwe->class) { 418 + case XE_ENGINE_CLASS_VIDEO_DECODE: 419 + slice = hwe->instance / 2; 420 + break; 421 + case XE_ENGINE_CLASS_VIDEO_ENHANCE: 422 + slice = hwe->instance; 423 + break; 424 + case XE_ENGINE_CLASS_OTHER: 425 + slice = 0; 426 + break; 427 + default: 428 + xe_gt_assert_msg(gt, false, 429 + "unknown media gt class %u (%s) during EGS setup\n", 430 + hwe->class, hwe->name); 431 + slice = 0; 432 + } 433 + 434 + values[slice_to_group[slice]].engines[guc_class] |= BIT(hwe->logical_instance); 435 + } 436 + 437 + *groups = values; 438 + *num_groups = group; 439 + } 440 + 441 + /** 442 + * xe_sriov_gt_pf_policy_has_sched_groups_support() - Checks whether scheduler 443 + * groups are supported. 444 + * @gt: the &xe_gt 445 + * 446 + * This function can only be called on PF. 447 + * 448 + * Return: true if scheduler groups are supported, false otherwise. 449 + */ 450 + bool xe_sriov_gt_pf_policy_has_sched_groups_support(struct xe_gt *gt) 451 + { 452 + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 453 + 454 + /* 455 + * The GuC supports scheduler groups from v70.53.0, but a fix for it has 456 + * been merged in v70.55.1, so we require the latter. The feature is 457 + * also only enabled on BMG and newer FW. 458 + */ 459 + return GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 55, 1) && 460 + gt_to_xe(gt)->info.platform >= XE_BATTLEMAGE; 461 + } 462 + 463 + static void pf_init_sched_groups(struct xe_gt *gt) 464 + { 465 + enum xe_sriov_sched_group_modes m; 466 + 467 + if (!xe_sriov_gt_pf_policy_has_sched_groups_support(gt)) 468 + return; 469 + 470 + /* 471 + * The GuC interface supports up to 8 groups. However, the GuC only 472 + * fully allocates resources for a subset of groups, based on the number 473 + * of engines and expected usage. The plan is for this to become 474 + * queryable via H2G, but for now GuC FW for all devices supports a 475 + * maximum of 2 groups so we can just hardcode that. 476 + */ 477 + gt->sriov.pf.policy.guc.sched_groups.max_groups = 2; 478 + 479 + for (m = XE_SRIOV_SCHED_GROUPS_DISABLED + 1; m < XE_SRIOV_SCHED_GROUPS_MODES_COUNT; m++) { 480 + u32 *num_groups = &gt->sriov.pf.policy.guc.sched_groups.modes[m].num_groups; 481 + struct guc_sched_group **groups = 482 + &gt->sriov.pf.policy.guc.sched_groups.modes[m].groups; 483 + 484 + switch (m) { 485 + case XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: 486 + /* this mode only has groups on the media GT */ 487 + if (xe_gt_is_media_type(gt)) 488 + pf_sched_group_media_slices(gt, groups, num_groups); 489 + break; 490 + case XE_SRIOV_SCHED_GROUPS_DISABLED: 491 + case XE_SRIOV_SCHED_GROUPS_MODES_COUNT: 492 + /* 493 + * By defining m of type enum xe_sriov_sched_group_modes 494 + * we can get the compiler to automatically flag 495 + * missing cases if new enum entries are added. However, 496 + * to keep the compiler happy we also need to add the 497 + * cases that are excluded from the loop. 498 + */ 499 + xe_gt_assert(gt, false); 500 + break; 501 + } 502 + 503 + xe_gt_assert(gt, *num_groups < GUC_MAX_SCHED_GROUPS); 504 + 505 + if (*num_groups) 506 + gt->sriov.pf.policy.guc.sched_groups.supported_modes |= BIT(m); 507 + } 508 + } 509 + 510 + /** 511 + * xe_sriov_gt_pf_policy_has_multi_group_modes() - check whether the GT supports 512 + * any scheduler modes that have multiple groups 513 + * @gt: the &xe_gt to check 514 + * 515 + * This function can only be called on PF. 516 + * 517 + * Return: true if the GT supports modes with multiple groups, false otherwise. 518 + */ 519 + bool xe_sriov_gt_pf_policy_has_multi_group_modes(struct xe_gt *gt) 520 + { 521 + return gt->sriov.pf.policy.guc.sched_groups.supported_modes; 522 + } 523 + 524 + /** 525 + * xe_sriov_gt_pf_policy_has_sched_group_mode() - check whether the GT supports 526 + * a specific scheduler group mode 527 + * @gt: the &xe_gt to check 528 + * @mode: the mode to check 529 + * 530 + * This function can only be called on PF. 531 + * 532 + * Return: true if the GT supports the specified mode, false otherwise. 533 + */ 534 + bool xe_sriov_gt_pf_policy_has_sched_group_mode(struct xe_gt *gt, 535 + enum xe_sriov_sched_group_modes mode) 536 + { 537 + if (mode == XE_SRIOV_SCHED_GROUPS_DISABLED) 538 + return true; 539 + 540 + return gt->sriov.pf.policy.guc.sched_groups.supported_modes & BIT(mode); 541 + } 542 + 543 + static int __pf_provision_sched_groups(struct xe_gt *gt, enum xe_sriov_sched_group_modes mode) 544 + { 545 + struct guc_sched_group *groups = gt->sriov.pf.policy.guc.sched_groups.modes[mode].groups; 546 + u32 num_groups = gt->sriov.pf.policy.guc.sched_groups.modes[mode].num_groups; 547 + 548 + return pf_push_policy_payload(gt, GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY, 549 + groups, num_groups * GUC_MAX_ENGINE_CLASSES); 550 + } 551 + 552 + static int pf_provision_sched_groups(struct xe_gt *gt, enum xe_sriov_sched_group_modes mode) 553 + { 554 + int err; 555 + 556 + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 557 + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); 558 + 559 + if (!xe_sriov_gt_pf_policy_has_sched_group_mode(gt, mode)) 560 + return -EINVAL; 561 + 562 + /* already in the desired mode */ 563 + if (gt->sriov.pf.policy.guc.sched_groups.current_mode == mode) 564 + return 0; 565 + 566 + /* 567 + * We don't allow changing this with VFs active since it is hard for 568 + * VFs to check. 569 + */ 570 + if (xe_sriov_pf_num_vfs(gt_to_xe(gt))) 571 + return -EBUSY; 572 + 573 + /* 574 + * The GuC silently ignores the setting if any MLRC contexts are 575 + * registered. We expect the admin to make sure that all apps that use 576 + * MLRC are terminated before scheduler groups are enabled, so this 577 + * check is just to make sure that the exec_queue destruction has been 578 + * completed. 579 + */ 580 + if (mode != XE_SRIOV_SCHED_GROUPS_DISABLED && 581 + xe_guc_has_registered_mlrc_queues(&gt->uc.guc)) { 582 + xe_gt_sriov_notice(gt, "can't enable sched groups with active MLRC queues\n"); 583 + return -EPERM; 584 + } 585 + 586 + err = __pf_provision_sched_groups(gt, mode); 587 + if (err) 588 + return err; 589 + 590 + gt->sriov.pf.policy.guc.sched_groups.current_mode = mode; 591 + 592 + return 0; 593 + } 594 + 595 + static int pf_reprovision_sched_groups(struct xe_gt *gt) 596 + { 597 + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 598 + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); 599 + 600 + /* We only have something to provision if we have possible groups */ 601 + if (!xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) 602 + return 0; 603 + 604 + return __pf_provision_sched_groups(gt, gt->sriov.pf.policy.guc.sched_groups.current_mode); 605 + } 606 + 607 + static void pf_sanitize_sched_groups(struct xe_gt *gt) 608 + { 609 + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 610 + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); 611 + 612 + gt->sriov.pf.policy.guc.sched_groups.current_mode = XE_SRIOV_SCHED_GROUPS_DISABLED; 613 + } 614 + 615 + /** 616 + * xe_gt_sriov_pf_policy_set_sched_groups_mode() - Control the 'sched_groups' policy. 617 + * @gt: the &xe_gt where to apply the policy 618 + * @mode: the sched_group mode to be activated 619 + * 620 + * This function can only be called on PF. 621 + * 622 + * Return: 0 on success or a negative error code on failure. 623 + */ 624 + int xe_gt_sriov_pf_policy_set_sched_groups_mode(struct xe_gt *gt, 625 + enum xe_sriov_sched_group_modes mode) 626 + { 627 + if (!xe_sriov_gt_pf_policy_has_multi_group_modes(gt)) 628 + return -ENODEV; 629 + 630 + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); 631 + return pf_provision_sched_groups(gt, mode); 632 + } 633 + 634 + /** 635 + * xe_gt_sriov_pf_policy_sched_groups_enabled() - check whether the GT has 636 + * multiple scheduler groups enabled 637 + * @gt: the &xe_gt to check 638 + * 639 + * This function can only be called on PF. 640 + * 641 + * Return: true if the GT has multiple groups enabled, false otherwise. 642 + */ 643 + bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt) 644 + { 645 + return gt->sriov.pf.policy.guc.sched_groups.current_mode != XE_SRIOV_SCHED_GROUPS_DISABLED; 646 + } 647 + 375 648 static void pf_sanitize_guc_policies(struct xe_gt *gt) 376 649 { 377 650 pf_sanitize_sched_if_idle(gt); 378 651 pf_sanitize_reset_engine(gt); 379 652 pf_sanitize_sample_period(gt); 653 + pf_sanitize_sched_groups(gt); 380 654 } 381 655 382 656 /** ··· 710 394 err |= pf_reprovision_sched_if_idle(gt); 711 395 err |= pf_reprovision_reset_engine(gt); 712 396 err |= pf_reprovision_sample_period(gt); 397 + err |= pf_reprovision_sched_groups(gt); 713 398 mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); 714 399 715 400 xe_pm_runtime_put(gt_to_xe(gt)); 716 401 717 402 return err ? -ENXIO : 0; 403 + } 404 + 405 + /** 406 + * xe_gt_sriov_pf_policy_init() - Initializes the SW state of the PF policies. 407 + * @gt: the &xe_gt 408 + * 409 + * This function can only be called on PF. This function does not touch the HW, 410 + * but must be called after the engines have been initialized. 411 + */ 412 + void xe_gt_sriov_pf_policy_init(struct xe_gt *gt) 413 + { 414 + pf_init_sched_groups(gt); 718 415 } 719 416 720 417 static void print_guc_policies(struct drm_printer *p, struct xe_gt_sriov_guc_policies *policy)

+10

drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h

··· 8 8 9 9 #include <linux/types.h> 10 10 11 + #include "xe_gt_sriov_pf_policy_types.h" 12 + 11 13 struct drm_printer; 12 14 struct xe_gt; 13 15 ··· 19 17 bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt); 20 18 int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value); 21 19 u32 xe_gt_sriov_pf_policy_get_sample_period(struct xe_gt *gt); 20 + bool xe_sriov_gt_pf_policy_has_sched_groups_support(struct xe_gt *gt); 21 + bool xe_sriov_gt_pf_policy_has_multi_group_modes(struct xe_gt *gt); 22 + bool xe_sriov_gt_pf_policy_has_sched_group_mode(struct xe_gt *gt, 23 + enum xe_sriov_sched_group_modes mode); 24 + int xe_gt_sriov_pf_policy_set_sched_groups_mode(struct xe_gt *gt, 25 + enum xe_sriov_sched_group_modes mode); 26 + bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt); 22 27 28 + void xe_gt_sriov_pf_policy_init(struct xe_gt *gt); 23 29 void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt); 24 30 int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset); 25 31 int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p);

+39

drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h

··· 8 8 9 9 #include <linux/types.h> 10 10 11 + #include "abi/guc_scheduler_abi.h" 12 + 13 + /** 14 + * enum xe_sriov_sched_group_modes - list of possible scheduler group modes 15 + * @XE_SRIOV_SCHED_GROUPS_DISABLED: no separate groups (i.e., all engines in group 0) 16 + * @XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES: separate groups for each media slice 17 + * @XE_SRIOV_SCHED_GROUPS_MODES_COUNT: number of valid modes 18 + */ 19 + enum xe_sriov_sched_group_modes { 20 + XE_SRIOV_SCHED_GROUPS_DISABLED = 0, 21 + XE_SRIOV_SCHED_GROUPS_MEDIA_SLICES, 22 + XE_SRIOV_SCHED_GROUPS_MODES_COUNT /* must be last */ 23 + }; 24 + 25 + /** 26 + * struct xe_gt_sriov_scheduler_groups - Scheduler groups policy info 27 + * @max_groups: max number of groups supported by the GuC for the platform 28 + * @supported_modes: mask of supported modes 29 + * @current_mode: active scheduler groups mode 30 + * @modes: array of masks and their number for each mode 31 + * @modes.groups: array of engine instance groups in given mode, with each group 32 + * consisting of GUC_MAX_ENGINE_CLASSES engine instances masks. A 33 + * A NULL value indicates that all the engines are in the same 34 + * group for this mode on this GT. 35 + * @modes.num_groups: number of groups in given mode, zero if all the engines 36 + * are in the same group. 37 + */ 38 + struct xe_gt_sriov_scheduler_groups { 39 + u8 max_groups; 40 + u32 supported_modes; 41 + enum xe_sriov_sched_group_modes current_mode; 42 + struct { 43 + struct guc_sched_group *groups; 44 + u32 num_groups; 45 + } modes[XE_SRIOV_SCHED_GROUPS_MODES_COUNT]; 46 + }; 47 + 11 48 /** 12 49 * struct xe_gt_sriov_guc_policies - GuC SR-IOV policies. 13 50 * @sched_if_idle: controls strict scheduling policy. 14 51 * @reset_engine: controls engines reset on VF switch policy. 15 52 * @sample_period: adverse events sampling period (in milliseconds). 53 + * @sched_groups: available scheduling group configurations. 16 54 */ 17 55 struct xe_gt_sriov_guc_policies { 18 56 bool sched_if_idle; 19 57 bool reset_engine; 20 58 u32 sample_period; 59 + struct xe_gt_sriov_scheduler_groups sched_groups; 21 60 }; 22 61 23 62 /**

+67

drivers/gpu/drm/xe/xe_gt_sriov_vf.c

··· 612 612 gt->sriov.vf.runtime.gmdid = xe_gt_sriov_vf_gmdid(gt); 613 613 } 614 614 615 + static int vf_query_sched_groups(struct xe_gt *gt) 616 + { 617 + struct xe_guc *guc = &gt->uc.guc; 618 + struct xe_uc_fw_version guc_version; 619 + u32 value = 0; 620 + int err; 621 + 622 + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); 623 + 624 + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 26, 0)) 625 + return 0; 626 + 627 + err = guc_action_query_single_klv32(guc, 628 + GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY, 629 + &value); 630 + if (unlikely(err)) { 631 + xe_gt_sriov_err(gt, "Failed to obtain sched groups status (%pe)\n", 632 + ERR_PTR(err)); 633 + return err; 634 + } 635 + 636 + /* valid values are 0 (disabled) and 1 (enabled) */ 637 + if (value > 1) { 638 + xe_gt_sriov_err(gt, "Invalid sched groups status %u\n", value); 639 + return -EPROTO; 640 + } 641 + 642 + xe_gt_sriov_dbg(gt, "sched groups %s\n", str_enabled_disabled(value)); 643 + return value; 644 + } 645 + 646 + static int vf_cache_sched_groups_status(struct xe_gt *gt) 647 + { 648 + int ret; 649 + 650 + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); 651 + 652 + ret = vf_query_sched_groups(gt); 653 + if (ret < 0) 654 + return ret; 655 + 656 + gt->sriov.vf.runtime.uses_sched_groups = ret; 657 + 658 + return 0; 659 + } 660 + 615 661 /** 616 662 * xe_gt_sriov_vf_query_config - Query SR-IOV config data over MMIO. 617 663 * @gt: the &xe_gt ··· 687 641 if (unlikely(err)) 688 642 return err; 689 643 644 + err = vf_cache_sched_groups_status(gt); 645 + if (unlikely(err)) 646 + return err; 647 + 690 648 if (has_gmdid(xe)) 691 649 vf_cache_gmdid(gt); 692 650 693 651 return 0; 652 + } 653 + 654 + /** 655 + * xe_gt_sriov_vf_sched_groups_enabled() - Check if PF has enabled multiple 656 + * scheduler groups 657 + * @gt: the &xe_gt 658 + * 659 + * This function is for VF use only. 660 + * 661 + * Return: true if shed groups were enabled, false otherwise. 662 + */ 663 + bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt) 664 + { 665 + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); 666 + xe_gt_assert(gt, gt->sriov.vf.guc_version.major); 667 + 668 + return gt->sriov.vf.runtime.uses_sched_groups; 694 669 } 695 670 696 671 /**

+1

drivers/gpu/drm/xe/xe_gt_sriov_vf.h

··· 30 30 u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt); 31 31 u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt); 32 32 u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt); 33 + bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt); 33 34 34 35 u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg); 35 36 void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);

+2

drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h

··· 27 27 struct xe_gt_sriov_vf_runtime { 28 28 /** @gmdid: cached value of the GDMID register. */ 29 29 u32 gmdid; 30 + /** @uses_sched_groups: whether PF enabled sched groups or not. */ 31 + bool uses_sched_groups; 30 32 /** @regs_size: size of runtime register array. */ 31 33 u32 regs_size; 32 34 /** @num_regs: number of runtime registers in the array. */

+1 -1

drivers/gpu/drm/xe/xe_guc.c

··· 175 175 * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines 176 176 * to move management back to the GuC. 177 177 */ 178 - if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 178 + if (CCS_INSTANCES(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 179 179 return true; 180 180 181 181 return false;

+1 -1

drivers/gpu/drm/xe/xe_guc_capture.h

··· 8 8 9 9 #include <linux/types.h> 10 10 #include "abi/guc_capture_abi.h" 11 + #include "abi/guc_scheduler_abi.h" 11 12 #include "xe_guc.h" 12 - #include "xe_guc_fwif.h" 13 13 14 14 struct xe_exec_queue; 15 15 struct xe_guc;

+16 -5

drivers/gpu/drm/xe/xe_guc_ct.c

··· 206 206 { 207 207 g2h_fence->cancel = true; 208 208 g2h_fence->fail = true; 209 - g2h_fence->done = true; 209 + 210 + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 211 + WRITE_ONCE(g2h_fence->done, true); 210 212 } 211 213 212 214 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) ··· 529 527 if (ct->g2h_outstanding) 530 528 xe_pm_runtime_put(ct_to_xe(ct)); 531 529 ct->g2h_outstanding = 0; 532 - ct->state = state; 530 + 531 + /* 532 + * WRITE_ONCE pairs with READ_ONCEs in xe_guc_ct_initialized and 533 + * xe_guc_ct_enabled. 534 + */ 535 + WRITE_ONCE(ct->state, state); 533 536 534 537 xe_gt_dbg(gt, "GuC CT communication channel %s\n", 535 538 state == XE_GUC_CT_STATE_STOPPED ? "stopped" : ··· 1301 1294 return ret; 1302 1295 } 1303 1296 1304 - ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1297 + /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response 1298 + * and g2h_fence_cancel. 1299 + */ 1300 + ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); 1305 1301 if (!ret) { 1306 1302 LNL_FLUSH_WORK(&ct->g2h_worker); 1307 - if (g2h_fence.done) { 1303 + if (READ_ONCE(g2h_fence.done)) { 1308 1304 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1309 1305 g2h_fence.seqno, action[0]); 1310 1306 ret = 1; ··· 1508 1498 1509 1499 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1510 1500 1511 - g2h_fence->done = true; 1501 + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 1502 + WRITE_ONCE(g2h_fence->done, true); 1512 1503 smp_mb(); 1513 1504 1514 1505 wake_up_all(&ct->g2h_fence_wq);

+4 -2

drivers/gpu/drm/xe/xe_guc_ct.h

··· 30 30 31 31 static inline bool xe_guc_ct_initialized(struct xe_guc_ct *ct) 32 32 { 33 - return ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED; 33 + /* READ_ONCE pairs with WRITE_ONCE in guc_ct_change_state. */ 34 + return READ_ONCE(ct->state) != XE_GUC_CT_STATE_NOT_INITIALIZED; 34 35 } 35 36 36 37 static inline bool xe_guc_ct_enabled(struct xe_guc_ct *ct) 37 38 { 38 - return ct->state == XE_GUC_CT_STATE_ENABLED; 39 + /* READ_ONCE pairs with WRITE_ONCE in guc_ct_change_state. */ 40 + return READ_ONCE(ct->state) == XE_GUC_CT_STATE_ENABLED; 39 41 } 40 42 41 43 static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct)

+1 -53

drivers/gpu/drm/xe/xe_guc_fwif.h

··· 10 10 11 11 #include "abi/guc_capture_abi.h" 12 12 #include "abi/guc_klvs_abi.h" 13 + #include "abi/guc_scheduler_abi.h" 13 14 #include "xe_hw_engine_types.h" 14 15 15 16 #define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 4 ··· 19 18 #define G2H_LEN_DW_G2G_NOTIFY_MIN 3 20 19 #define G2H_LEN_DW_MULTI_QUEUE_CONTEXT 3 21 20 #define G2H_LEN_DW_PAGE_RECLAMATION 3 22 - 23 - #define GUC_ID_MAX 65535 24 - #define GUC_ID_UNKNOWN 0xffffffff 25 - 26 - #define GUC_CONTEXT_DISABLE 0 27 - #define GUC_CONTEXT_ENABLE 1 28 - 29 - #define GUC_CLIENT_PRIORITY_KMD_HIGH 0 30 - #define GUC_CLIENT_PRIORITY_HIGH 1 31 - #define GUC_CLIENT_PRIORITY_KMD_NORMAL 2 32 - #define GUC_CLIENT_PRIORITY_NORMAL 3 33 - #define GUC_CLIENT_PRIORITY_NUM 4 34 - 35 - #define GUC_RENDER_ENGINE 0 36 - #define GUC_VIDEO_ENGINE 1 37 - #define GUC_BLITTER_ENGINE 2 38 - #define GUC_VIDEOENHANCE_ENGINE 3 39 - #define GUC_VIDEO_ENGINE2 4 40 - #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) 41 - 42 - #define GUC_RENDER_CLASS 0 43 - #define GUC_VIDEO_CLASS 1 44 - #define GUC_VIDEOENHANCE_CLASS 2 45 - #define GUC_BLITTER_CLASS 3 46 - #define GUC_COMPUTE_CLASS 4 47 - #define GUC_GSC_OTHER_CLASS 5 48 - #define GUC_LAST_ENGINE_CLASS GUC_GSC_OTHER_CLASS 49 - #define GUC_MAX_ENGINE_CLASSES 16 50 - #define GUC_MAX_INSTANCES_PER_CLASS 32 51 - 52 - #define GUC_CONTEXT_NORMAL 0 53 - #define GUC_CONTEXT_COMPRESSION_SAVE 1 54 - #define GUC_CONTEXT_COMPRESSION_RESTORE 2 55 - #define GUC_CONTEXT_COUNT (GUC_CONTEXT_COMPRESSION_RESTORE + 1) 56 - 57 - /* Helper for context registration H2G */ 58 - struct guc_ctxt_registration_info { 59 - u32 flags; 60 - u32 context_idx; 61 - u32 engine_class; 62 - u32 engine_submit_mask; 63 - u32 wq_desc_lo; 64 - u32 wq_desc_hi; 65 - u32 wq_base_lo; 66 - u32 wq_base_hi; 67 - u32 wq_size; 68 - u32 cgp_lo; 69 - u32 cgp_hi; 70 - u32 hwlrca_lo; 71 - u32 hwlrca_hi; 72 - }; 73 - #define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) 74 - #define CONTEXT_REGISTRATION_FLAG_TYPE GENMASK(2, 1) 75 21 76 22 /* 32-bit KLV structure as used by policy updates and others */ 77 23 struct guc_klv_generic_dw_t {

+9

drivers/gpu/drm/xe/xe_guc_klv_helpers.c

··· 21 21 const char *xe_guc_klv_key_to_string(u16 key) 22 22 { 23 23 switch (key) { 24 + /* GuC Global Config KLVs */ 25 + case GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY: 26 + return "group_scheduling_available"; 24 27 /* VGT POLICY keys */ 25 28 case GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY: 26 29 return "sched_if_idle"; 27 30 case GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY: 28 31 return "sample_period"; 32 + case GUC_KLV_VGT_POLICY_ENGINE_GROUP_CONFIG_KEY: 33 + return "engine_group_config"; 29 34 case GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY: 30 35 return "reset_engine"; 31 36 /* VF CFG keys */ ··· 56 51 return "begin_ctx_id"; 57 52 case GUC_KLV_VF_CFG_SCHED_PRIORITY_KEY: 58 53 return "sched_priority"; 54 + case GUC_KLV_VF_CFG_ENGINE_GROUP_EXEC_QUANTUM_KEY: 55 + return "sched_groups_exec_quantum"; 56 + case GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_KEY: 57 + return "sched_groups_preempt_timeout"; 59 58 60 59 /* VF CFG threshold keys */ 61 60 #define define_threshold_key_to_string_case(TAG, NAME, ...) \

+39 -1

drivers/gpu/drm/xe/xe_guc_submit.c

··· 407 407 408 408 *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV; 409 409 410 - if (CCS_MASK(guc_to_gt(guc))) 410 + if (CCS_INSTANCES(guc_to_gt(guc))) 411 411 emit = emit_render_compute_yield_klv(emit); 412 412 413 413 count = emit - data; ··· 672 672 set_exec_queue_banned(eq); 673 673 mutex_unlock(&group->list_lock); 674 674 } 675 + 676 + /* Helper for context registration H2G */ 677 + struct guc_ctxt_registration_info { 678 + u32 flags; 679 + u32 context_idx; 680 + u32 engine_class; 681 + u32 engine_submit_mask; 682 + u32 wq_desc_lo; 683 + u32 wq_desc_hi; 684 + u32 wq_base_lo; 685 + u32 wq_base_hi; 686 + u32 wq_size; 687 + u32 cgp_lo; 688 + u32 cgp_hi; 689 + u32 hwlrca_lo; 690 + u32 hwlrca_hi; 691 + }; 675 692 676 693 #define parallel_read(xe_, map_, field_) \ 677 694 xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ ··· 3561 3544 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 3562 3545 guc_exec_queue_print(q, p); 3563 3546 mutex_unlock(&guc->submission_state.lock); 3547 + } 3548 + 3549 + /** 3550 + * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues 3551 + * registered with the GuC 3552 + * @guc: GuC. 3553 + * 3554 + * Return: true if any MLRC queue is registered with the GuC, false otherwise. 3555 + */ 3556 + bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc) 3557 + { 3558 + struct xe_exec_queue *q; 3559 + unsigned long index; 3560 + 3561 + guard(mutex)(&guc->submission_state.lock); 3562 + 3563 + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 3564 + if (q->width > 1) 3565 + return true; 3566 + 3567 + return false; 3564 3568 } 3565 3569 3566 3570 /**

+2

drivers/gpu/drm/xe/xe_guc_submit.h

··· 52 52 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); 53 53 void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type); 54 54 55 + bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc); 56 + 55 57 int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); 56 58 57 59 #endif

+25 -7

drivers/gpu/drm/xe/xe_migrate.c

··· 35 35 #include "xe_sa.h" 36 36 #include "xe_sched_job.h" 37 37 #include "xe_sriov_vf_ccs.h" 38 + #include "xe_svm.h" 38 39 #include "xe_sync.h" 39 40 #include "xe_trace_bo.h" 40 41 #include "xe_validation.h" ··· 472 471 EXEC_QUEUE_FLAG_KERNEL | 473 472 EXEC_QUEUE_FLAG_PERMANENT | 474 473 EXEC_QUEUE_FLAG_HIGH_PRIORITY | 475 - EXEC_QUEUE_FLAG_MIGRATE, 0); 474 + EXEC_QUEUE_FLAG_MIGRATE | 475 + EXEC_QUEUE_FLAG_LOW_LATENCY, 0); 476 476 } else { 477 477 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 478 478 XE_ENGINE_CLASS_COPY, ··· 2050 2048 u64 pte; 2051 2049 2052 2050 xe_tile_assert(m->tile, sram_addr[i].proto == 2053 - DRM_INTERCONNECT_SYSTEM); 2051 + DRM_INTERCONNECT_SYSTEM || 2052 + sram_addr[i].proto == XE_INTERCONNECT_P2P); 2054 2053 xe_tile_assert(m->tile, addr); 2055 2054 xe_tile_assert(m->tile, PAGE_ALIGNED(addr)); 2056 2055 ··· 2116 2113 unsigned long sram_offset, 2117 2114 struct drm_pagemap_addr *sram_addr, 2118 2115 u64 vram_addr, 2116 + struct dma_fence *deps, 2119 2117 const enum xe_migrate_copy_dir dir) 2120 2118 { 2121 2119 struct xe_gt *gt = m->tile->primary_gt; ··· 2205 2201 2206 2202 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); 2207 2203 2204 + if (deps && !dma_fence_is_signaled(deps)) { 2205 + dma_fence_get(deps); 2206 + err = drm_sched_job_add_dependency(&job->drm, deps); 2207 + if (err) 2208 + dma_fence_wait(deps, false); 2209 + err = 0; 2210 + } 2211 + 2208 2212 mutex_lock(&m->job_mutex); 2209 2213 xe_sched_job_arm(job); 2210 2214 fence = dma_fence_get(&job->drm.s_fence->finished); ··· 2238 2226 * @npages: Number of pages to migrate. 2239 2227 * @src_addr: Array of DMA information (source of migrate) 2240 2228 * @dst_addr: Device physical address of VRAM (destination of migrate) 2229 + * @deps: struct dma_fence representing the dependencies that need 2230 + * to be signaled before migration. 2241 2231 * 2242 2232 * Copy from an array dma addresses to a VRAM device physical address 2243 2233 * ··· 2249 2235 struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, 2250 2236 unsigned long npages, 2251 2237 struct drm_pagemap_addr *src_addr, 2252 - u64 dst_addr) 2238 + u64 dst_addr, 2239 + struct dma_fence *deps) 2253 2240 { 2254 2241 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, 2255 - XE_MIGRATE_COPY_TO_VRAM); 2242 + deps, XE_MIGRATE_COPY_TO_VRAM); 2256 2243 } 2257 2244 2258 2245 /** ··· 2262 2247 * @npages: Number of pages to migrate. 2263 2248 * @src_addr: Device physical address of VRAM (source of migrate) 2264 2249 * @dst_addr: Array of DMA information (destination of migrate) 2250 + * @deps: struct dma_fence representing the dependencies that need 2251 + * to be signaled before migration. 2265 2252 * 2266 2253 * Copy from a VRAM device physical address to an array dma addresses 2267 2254 * ··· 2273 2256 struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, 2274 2257 unsigned long npages, 2275 2258 u64 src_addr, 2276 - struct drm_pagemap_addr *dst_addr) 2259 + struct drm_pagemap_addr *dst_addr, 2260 + struct dma_fence *deps) 2277 2261 { 2278 2262 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, 2279 - XE_MIGRATE_COPY_TO_SRAM); 2263 + deps, XE_MIGRATE_COPY_TO_SRAM); 2280 2264 } 2281 2265 2282 2266 static void xe_migrate_dma_unmap(struct xe_device *xe, ··· 2453 2435 __fence = xe_migrate_vram(m, current_bytes, 2454 2436 (unsigned long)buf & ~PAGE_MASK, 2455 2437 &pagemap_addr[current_page], 2456 - vram_addr, write ? 2438 + vram_addr, NULL, write ? 2457 2439 XE_MIGRATE_COPY_TO_VRAM : 2458 2440 XE_MIGRATE_COPY_TO_SRAM); 2459 2441 if (IS_ERR(__fence)) {

+4 -2

drivers/gpu/drm/xe/xe_migrate.h

··· 116 116 struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, 117 117 unsigned long npages, 118 118 struct drm_pagemap_addr *src_addr, 119 - u64 dst_addr); 119 + u64 dst_addr, 120 + struct dma_fence *deps); 120 121 121 122 struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, 122 123 unsigned long npages, 123 124 u64 src_addr, 124 - struct drm_pagemap_addr *dst_addr); 125 + struct drm_pagemap_addr *dst_addr, 126 + struct dma_fence *deps); 125 127 126 128 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 127 129 struct xe_bo *src_bo,

+7 -1

drivers/gpu/drm/xe/xe_pci.c

··· 27 27 #include "xe_macros.h" 28 28 #include "xe_mmio.h" 29 29 #include "xe_module.h" 30 + #include "xe_pci_rebar.h" 30 31 #include "xe_pci_sriov.h" 31 32 #include "xe_pci_types.h" 32 33 #include "xe_pm.h" ··· 371 370 .has_i2c = true, 372 371 .has_late_bind = true, 373 372 .has_pre_prod_wa = 1, 373 + .has_soc_remapper_telem = true, 374 374 .has_sriov = true, 375 375 .has_mem_copy_instr = true, 376 376 .max_gt_per_tile = 2, ··· 423 421 .has_mbx_power_limits = true, 424 422 .has_mert = true, 425 423 .has_pre_prod_wa = 1, 424 + .has_soc_remapper_sysctrl = true, 425 + .has_soc_remapper_telem = true, 426 426 .has_sriov = true, 427 427 .max_gt_per_tile = 2, 428 428 .require_force_probe = true, ··· 696 692 xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; 697 693 xe->info.has_pre_prod_wa = desc->has_pre_prod_wa; 698 694 xe->info.has_pxp = desc->has_pxp; 695 + xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl; 696 + xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; 699 697 xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && 700 698 desc->has_sriov; 701 699 xe->info.has_mem_copy_instr = desc->has_mem_copy_instr; ··· 1022 1016 if (err) 1023 1017 return err; 1024 1018 1025 - xe_vram_resize_bar(xe); 1019 + xe_pci_rebar_resize(xe); 1026 1020 1027 1021 err = xe_device_probe_early(xe); 1028 1022 /*

+106

drivers/gpu/drm/xe/xe_pci_rebar.c

··· 1 + // SPDX-License-Identifier: MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #include <linux/pci.h> 7 + #include <linux/types.h> 8 + 9 + #include "regs/xe_bars.h" 10 + #include "xe_device_types.h" 11 + #include "xe_module.h" 12 + #include "xe_pci_rebar.h" 13 + #include "xe_printk.h" 14 + 15 + static void resize_bar(struct xe_device *xe, int resno, resource_size_t size) 16 + { 17 + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 18 + int bar_size = pci_rebar_bytes_to_size(size); 19 + int ret; 20 + 21 + ret = pci_resize_resource(pdev, resno, bar_size, 0); 22 + if (ret) { 23 + xe_info(xe, "Failed to resize BAR%d to %dMiB (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", 24 + resno, 1 << bar_size, ERR_PTR(ret)); 25 + return; 26 + } 27 + 28 + xe_info(xe, "BAR%d resized to %dMiB\n", resno, 1 << bar_size); 29 + } 30 + 31 + /* 32 + * xe_pci_rebar_resize - Resize the LMEMBAR 33 + * @xe: xe device instance 34 + * 35 + * If vram_bar_size module param is set, attempt to set to the requested size 36 + * else set to maximum possible size. 37 + */ 38 + void xe_pci_rebar_resize(struct xe_device *xe) 39 + { 40 + int force_vram_bar_size = xe_modparam.force_vram_bar_size; 41 + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 42 + struct pci_bus *root = pdev->bus; 43 + resource_size_t current_size; 44 + resource_size_t rebar_size; 45 + struct resource *root_res; 46 + int max_size, i; 47 + u32 pci_cmd; 48 + 49 + /* gather some relevant info */ 50 + current_size = pci_resource_len(pdev, LMEM_BAR); 51 + 52 + if (force_vram_bar_size < 0) 53 + return; 54 + 55 + /* set to a specific size? */ 56 + if (force_vram_bar_size) { 57 + rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * 58 + (resource_size_t)SZ_1M); 59 + 60 + if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { 61 + xe_info(xe, "Requested size %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", 62 + (u64)pci_rebar_size_to_bytes(rebar_size) >> ilog2(SZ_1M), 63 + pci_rebar_get_possible_sizes(pdev, LMEM_BAR), 64 + (u64)current_size >> ilog2(SZ_1M)); 65 + return; 66 + } 67 + 68 + rebar_size = pci_rebar_size_to_bytes(rebar_size); 69 + if (rebar_size == current_size) 70 + return; 71 + } else { 72 + max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); 73 + if (max_size < 0) 74 + return; 75 + rebar_size = pci_rebar_size_to_bytes(max_size); 76 + 77 + /* only resize if larger than current */ 78 + if (rebar_size <= current_size) 79 + return; 80 + } 81 + 82 + xe_info(xe, "Attempting to resize bar from %lluMiB -> %lluMiB\n", 83 + (u64)current_size >> ilog2(SZ_1M), (u64)rebar_size >> ilog2(SZ_1M)); 84 + 85 + while (root->parent) 86 + root = root->parent; 87 + 88 + pci_bus_for_each_resource(root, root_res, i) { 89 + if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 90 + (u64)root_res->start > 0x100000000ul) 91 + break; 92 + } 93 + 94 + if (!root_res) { 95 + xe_info(xe, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); 96 + return; 97 + } 98 + 99 + pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); 100 + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); 101 + 102 + resize_bar(xe, LMEM_BAR, rebar_size); 103 + 104 + pci_assign_unassigned_bus_resources(pdev->bus); 105 + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); 106 + }

+13

drivers/gpu/drm/xe/xe_pci_rebar.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef _XE_PCI_REBAR_H_ 7 + #define _XE_PCI_REBAR_H_ 8 + 9 + struct xe_device; 10 + 11 + void xe_pci_rebar_resize(struct xe_device *xe); 12 + 13 + #endif

+2

drivers/gpu/drm/xe/xe_pci_types.h

··· 53 53 u8 has_pre_prod_wa:1; 54 54 u8 has_page_reclaim_hw_assist:1; 55 55 u8 has_pxp:1; 56 + u8 has_soc_remapper_sysctrl:1; 57 + u8 has_soc_remapper_telem:1; 56 58 u8 has_sriov:1; 57 59 u8 needs_scratch:1; 58 60 u8 skip_guc_pc:1;

+52

drivers/gpu/drm/xe/xe_soc_remapper.c

··· 1 + // SPDX-License-Identifier: MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #include "regs/xe_soc_remapper_regs.h" 7 + #include "xe_mmio.h" 8 + #include "xe_soc_remapper.h" 9 + 10 + static void xe_soc_remapper_set_region(struct xe_device *xe, struct xe_reg reg, 11 + u32 mask, u32 val) 12 + { 13 + guard(spinlock_irqsave)(&xe->soc_remapper.lock); 14 + xe_mmio_rmw32(xe_root_tile_mmio(xe), reg, mask, val); 15 + } 16 + 17 + static void xe_soc_remapper_set_telem_region(struct xe_device *xe, u32 index) 18 + { 19 + xe_soc_remapper_set_region(xe, SG_REMAP_INDEX1, SG_REMAP_TELEM_MASK, 20 + REG_FIELD_PREP(SG_REMAP_TELEM_MASK, index)); 21 + } 22 + 23 + static void xe_soc_remapper_set_sysctrl_region(struct xe_device *xe, u32 index) 24 + { 25 + xe_soc_remapper_set_region(xe, SG_REMAP_INDEX1, SG_REMAP_SYSCTRL_MASK, 26 + REG_FIELD_PREP(SG_REMAP_SYSCTRL_MASK, index)); 27 + } 28 + 29 + /** 30 + * xe_soc_remapper_init() - Initialize SoC remapper 31 + * @xe: Pointer to xe device. 32 + * 33 + * Initialize SoC remapper. 34 + * 35 + * Return: 0 on success, error code on failure 36 + */ 37 + int xe_soc_remapper_init(struct xe_device *xe) 38 + { 39 + bool has_soc_remapper = xe->info.has_soc_remapper_telem || 40 + xe->info.has_soc_remapper_sysctrl; 41 + 42 + if (has_soc_remapper) 43 + spin_lock_init(&xe->soc_remapper.lock); 44 + 45 + if (xe->info.has_soc_remapper_telem) 46 + xe->soc_remapper.set_telem_region = xe_soc_remapper_set_telem_region; 47 + 48 + if (xe->info.has_soc_remapper_sysctrl) 49 + xe->soc_remapper.set_sysctrl_region = xe_soc_remapper_set_sysctrl_region; 50 + 51 + return 0; 52 + }

+13

drivers/gpu/drm/xe/xe_soc_remapper.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef _XE_SOC_REMAPPER_H_ 7 + #define _XE_SOC_REMAPPER_H_ 8 + 9 + #include "xe_device_types.h" 10 + 11 + int xe_soc_remapper_init(struct xe_device *xe); 12 + 13 + #endif

+610 -117

drivers/gpu/drm/xe/xe_svm.c

··· 3 3 * Copyright © 2024 Intel Corporation 4 4 */ 5 5 6 + #include <linux/pci-p2pdma.h> 7 + 6 8 #include <drm/drm_drv.h> 9 + #include <drm/drm_managed.h> 10 + #include <drm/drm_pagemap.h> 11 + #include <drm/drm_pagemap_util.h> 7 12 8 13 #include "xe_bo.h" 9 14 #include "xe_exec_queue_types.h" ··· 23 18 #include "xe_vm.h" 24 19 #include "xe_vm_types.h" 25 20 #include "xe_vram_types.h" 21 + 22 + /* Identifies subclasses of struct drm_pagemap_peer */ 23 + #define XE_PEER_PAGEMAP ((void *)0ul) 24 + #define XE_PEER_VM ((void *)1ul) 25 + 26 + /** 27 + * DOC: drm_pagemap reference-counting in xe: 28 + * 29 + * In addition to the drm_pagemap internal reference counting by its zone 30 + * device data, the xe driver holds the following long-time references: 31 + * 32 + * - struct xe_pagemap: 33 + * The xe_pagemap struct derives from struct drm_pagemap and uses its 34 + * reference count. 35 + * - SVM-enabled VMs: 36 + * SVM-enabled VMs look up and keeps a reference to all xe_pagemaps on 37 + * the same device. 38 + * - VMAs: 39 + * vmas keep a reference on the drm_pagemap indicated by a gpu_madvise() 40 + * call. 41 + * 42 + * In addition, all drm_pagemap or xe_pagemap pointers where lifetime cannot 43 + * be guaranteed by a vma reference under the vm lock should keep a reference. 44 + * That includes the range->pages.dpagemap pointer. 45 + */ 46 + 47 + static int xe_svm_get_pagemaps(struct xe_vm *vm); 48 + 49 + void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) 50 + { 51 + return force_smem ? NULL : vm->svm.peer.owner; 52 + } 26 53 27 54 static bool xe_svm_range_in_vram(struct xe_svm_range *range) 28 55 { ··· 313 276 314 277 static void xe_vma_set_default_attributes(struct xe_vma *vma) 315 278 { 316 - vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE; 317 - vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES; 318 - vma->attr.pat_index = vma->attr.default_pat_index; 319 - vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED; 279 + struct xe_vma_mem_attr default_attr = { 280 + .preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, 281 + .preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES, 282 + .pat_index = vma->attr.default_pat_index, 283 + .atomic_access = DRM_XE_ATOMIC_UNDEFINED, 284 + }; 285 + 286 + xe_vma_mem_attr_copy(&vma->attr, &default_attr); 320 287 } 321 288 322 289 static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) ··· 431 390 432 391 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 433 392 434 - static struct xe_vram_region *page_to_vr(struct page *page) 393 + static struct xe_vram_region *xe_pagemap_to_vr(struct xe_pagemap *xpagemap) 435 394 { 436 - return container_of(page_pgmap(page), struct xe_vram_region, pagemap); 395 + return xpagemap->vr; 437 396 } 438 397 439 - static u64 xe_vram_region_page_to_dpa(struct xe_vram_region *vr, 440 - struct page *page) 398 + static struct xe_pagemap *xe_page_to_pagemap(struct page *page) 441 399 { 442 - u64 dpa; 400 + return container_of(page_pgmap(page), struct xe_pagemap, pagemap); 401 + } 402 + 403 + static struct xe_vram_region *xe_page_to_vr(struct page *page) 404 + { 405 + return xe_pagemap_to_vr(xe_page_to_pagemap(page)); 406 + } 407 + 408 + static u64 xe_page_to_dpa(struct page *page) 409 + { 410 + struct xe_pagemap *xpagemap = xe_page_to_pagemap(page); 411 + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); 412 + u64 hpa_base = xpagemap->hpa_base; 443 413 u64 pfn = page_to_pfn(page); 444 414 u64 offset; 415 + u64 dpa; 445 416 446 417 xe_assert(vr->xe, is_device_private_page(page)); 447 - xe_assert(vr->xe, (pfn << PAGE_SHIFT) >= vr->hpa_base); 418 + xe_assert(vr->xe, (pfn << PAGE_SHIFT) >= hpa_base); 448 419 449 - offset = (pfn << PAGE_SHIFT) - vr->hpa_base; 420 + offset = (pfn << PAGE_SHIFT) - hpa_base; 450 421 dpa = vr->dpa_base + offset; 451 422 452 423 return dpa; 424 + } 425 + 426 + static u64 xe_page_to_pcie(struct page *page) 427 + { 428 + struct xe_pagemap *xpagemap = xe_page_to_pagemap(page); 429 + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); 430 + 431 + return xe_page_to_dpa(page) - vr->dpa_base + vr->io_start; 453 432 } 454 433 455 434 enum xe_svm_copy_dir { ··· 533 472 534 473 static int xe_svm_copy(struct page **pages, 535 474 struct drm_pagemap_addr *pagemap_addr, 536 - unsigned long npages, const enum xe_svm_copy_dir dir) 475 + unsigned long npages, const enum xe_svm_copy_dir dir, 476 + struct dma_fence *pre_migrate_fence) 537 477 { 538 478 struct xe_vram_region *vr = NULL; 539 479 struct xe_gt *gt = NULL; ··· 571 509 continue; 572 510 573 511 if (!vr && spage) { 574 - vr = page_to_vr(spage); 512 + vr = xe_page_to_vr(spage); 575 513 gt = xe_migrate_exec_queue(vr->migrate)->gt; 576 514 xe = vr->xe; 577 515 } 578 - XE_WARN_ON(spage && page_to_vr(spage) != vr); 516 + XE_WARN_ON(spage && xe_page_to_vr(spage) != vr); 579 517 580 518 /* 581 519 * CPU page and device page valid, capture physical address on ··· 583 521 * device pages. 584 522 */ 585 523 if (pagemap_addr[i].addr && spage) { 586 - __vram_addr = xe_vram_region_page_to_dpa(vr, spage); 524 + __vram_addr = xe_page_to_dpa(spage); 587 525 if (vram_addr == XE_VRAM_ADDR_INVALID) { 588 526 vram_addr = __vram_addr; 589 527 pos = i; ··· 623 561 __fence = xe_migrate_from_vram(vr->migrate, 624 562 i - pos + incr, 625 563 vram_addr, 626 - &pagemap_addr[pos]); 564 + &pagemap_addr[pos], 565 + pre_migrate_fence); 627 566 } else { 628 567 vm_dbg(&xe->drm, 629 568 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", ··· 633 570 __fence = xe_migrate_to_vram(vr->migrate, 634 571 i - pos + incr, 635 572 &pagemap_addr[pos], 636 - vram_addr); 573 + vram_addr, 574 + pre_migrate_fence); 637 575 } 638 576 if (IS_ERR(__fence)) { 639 577 err = PTR_ERR(__fence); 640 578 goto err_out; 641 579 } 642 - 580 + pre_migrate_fence = NULL; 643 581 dma_fence_put(fence); 644 582 fence = __fence; 645 583 } ··· 663 599 vram_addr, (u64)pagemap_addr[pos].addr, 1); 664 600 __fence = xe_migrate_from_vram(vr->migrate, 1, 665 601 vram_addr, 666 - &pagemap_addr[pos]); 602 + &pagemap_addr[pos], 603 + pre_migrate_fence); 667 604 } else { 668 605 vm_dbg(&xe->drm, 669 606 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", 670 607 (u64)pagemap_addr[pos].addr, vram_addr, 1); 671 608 __fence = xe_migrate_to_vram(vr->migrate, 1, 672 609 &pagemap_addr[pos], 673 - vram_addr); 610 + vram_addr, 611 + pre_migrate_fence); 674 612 } 675 613 if (IS_ERR(__fence)) { 676 614 err = PTR_ERR(__fence); 677 615 goto err_out; 678 616 } 679 - 617 + pre_migrate_fence = NULL; 680 618 dma_fence_put(fence); 681 619 fence = __fence; 682 620 } ··· 691 625 dma_fence_wait(fence, false); 692 626 dma_fence_put(fence); 693 627 } 628 + if (pre_migrate_fence) 629 + dma_fence_wait(pre_migrate_fence, false); 694 630 695 631 /* 696 632 * XXX: We can't derive the GT here (or anywhere in this functions, but ··· 709 641 710 642 static int xe_svm_copy_to_devmem(struct page **pages, 711 643 struct drm_pagemap_addr *pagemap_addr, 712 - unsigned long npages) 644 + unsigned long npages, 645 + struct dma_fence *pre_migrate_fence) 713 646 { 714 - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM); 647 + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM, 648 + pre_migrate_fence); 715 649 } 716 650 717 651 static int xe_svm_copy_to_ram(struct page **pages, 718 652 struct drm_pagemap_addr *pagemap_addr, 719 - unsigned long npages) 653 + unsigned long npages, 654 + struct dma_fence *pre_migrate_fence) 720 655 { 721 - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM); 656 + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM, 657 + pre_migrate_fence); 722 658 } 723 659 724 660 static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) ··· 735 663 struct xe_bo *bo = to_xe_bo(devmem_allocation); 736 664 struct xe_device *xe = xe_bo_device(bo); 737 665 666 + dma_fence_put(devmem_allocation->pre_migrate_fence); 738 667 xe_bo_put_async(bo); 739 668 xe_pm_runtime_put(xe); 740 669 } 741 670 742 - static u64 block_offset_to_pfn(struct xe_vram_region *vr, u64 offset) 671 + static u64 block_offset_to_pfn(struct drm_pagemap *dpagemap, u64 offset) 743 672 { 744 - return PHYS_PFN(offset + vr->hpa_base); 673 + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); 674 + 675 + return PHYS_PFN(offset + xpagemap->hpa_base); 745 676 } 746 677 747 678 static struct drm_buddy *vram_to_buddy(struct xe_vram_region *vram) ··· 764 689 list_for_each_entry(block, blocks, link) { 765 690 struct xe_vram_region *vr = block->private; 766 691 struct drm_buddy *buddy = vram_to_buddy(vr); 767 - u64 block_pfn = block_offset_to_pfn(vr, drm_buddy_block_offset(block)); 692 + u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap, 693 + drm_buddy_block_offset(block)); 768 694 int i; 769 695 770 696 for (i = 0; i < drm_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i) ··· 782 706 .copy_to_ram = xe_svm_copy_to_ram, 783 707 }; 784 708 709 + #else 710 + static int xe_svm_get_pagemaps(struct xe_vm *vm) 711 + { 712 + return 0; 713 + } 785 714 #endif 786 715 787 716 static const struct drm_gpusvm_ops gpusvm_ops = { ··· 800 719 SZ_64K, 801 720 SZ_4K, 802 721 }; 722 + 723 + static void xe_pagemap_put(struct xe_pagemap *xpagemap) 724 + { 725 + drm_pagemap_put(&xpagemap->dpagemap); 726 + } 727 + 728 + static void xe_svm_put_pagemaps(struct xe_vm *vm) 729 + { 730 + struct xe_device *xe = vm->xe; 731 + struct xe_tile *tile; 732 + int id; 733 + 734 + for_each_tile(tile, xe, id) { 735 + struct xe_pagemap *xpagemap = vm->svm.pagemaps[id]; 736 + 737 + if (xpagemap) 738 + xe_pagemap_put(xpagemap); 739 + vm->svm.pagemaps[id] = NULL; 740 + } 741 + } 742 + 743 + static struct device *xe_peer_to_dev(struct drm_pagemap_peer *peer) 744 + { 745 + if (peer->private == XE_PEER_PAGEMAP) 746 + return container_of(peer, struct xe_pagemap, peer)->dpagemap.drm->dev; 747 + 748 + return container_of(peer, struct xe_vm, svm.peer)->xe->drm.dev; 749 + } 750 + 751 + static bool xe_has_interconnect(struct drm_pagemap_peer *peer1, 752 + struct drm_pagemap_peer *peer2) 753 + { 754 + struct device *dev1 = xe_peer_to_dev(peer1); 755 + struct device *dev2 = xe_peer_to_dev(peer2); 756 + 757 + if (dev1 == dev2) 758 + return true; 759 + 760 + return pci_p2pdma_distance(to_pci_dev(dev1), dev2, true) >= 0; 761 + } 762 + 763 + static DRM_PAGEMAP_OWNER_LIST_DEFINE(xe_owner_list); 803 764 804 765 /** 805 766 * xe_svm_init() - SVM initialize ··· 861 738 INIT_WORK(&vm->svm.garbage_collector.work, 862 739 xe_svm_garbage_collector_work_func); 863 740 741 + vm->svm.peer.private = XE_PEER_VM; 742 + err = drm_pagemap_acquire_owner(&vm->svm.peer, &xe_owner_list, 743 + xe_has_interconnect); 744 + if (err) 745 + return err; 746 + 747 + err = xe_svm_get_pagemaps(vm); 748 + if (err) { 749 + drm_pagemap_release_owner(&vm->svm.peer); 750 + return err; 751 + } 752 + 864 753 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM", &vm->xe->drm, 865 754 current->mm, 0, vm->size, 866 755 xe_modparam.svm_notifier_size * SZ_1M, 867 756 &gpusvm_ops, fault_chunk_sizes, 868 757 ARRAY_SIZE(fault_chunk_sizes)); 869 758 drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->lock); 759 + 760 + if (err) { 761 + xe_svm_put_pagemaps(vm); 762 + drm_pagemap_release_owner(&vm->svm.peer); 763 + return err; 764 + } 870 765 } else { 871 766 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", 872 767 &vm->xe->drm, NULL, 0, 0, 0, NULL, ··· 904 763 { 905 764 xe_assert(vm->xe, xe_vm_is_closed(vm)); 906 765 flush_work(&vm->svm.garbage_collector.work); 766 + xe_svm_put_pagemaps(vm); 767 + drm_pagemap_release_owner(&vm->svm.peer); 907 768 } 908 769 909 770 /** ··· 921 778 drm_gpusvm_fini(&vm->svm.gpusvm); 922 779 } 923 780 781 + static bool xe_svm_range_has_pagemap_locked(const struct xe_svm_range *range, 782 + const struct drm_pagemap *dpagemap) 783 + { 784 + return range->base.pages.dpagemap == dpagemap; 785 + } 786 + 787 + static bool xe_svm_range_has_pagemap(struct xe_svm_range *range, 788 + const struct drm_pagemap *dpagemap) 789 + { 790 + struct xe_vm *vm = range_to_vm(&range->base); 791 + bool ret; 792 + 793 + xe_svm_notifier_lock(vm); 794 + ret = xe_svm_range_has_pagemap_locked(range, dpagemap); 795 + xe_svm_notifier_unlock(vm); 796 + 797 + return ret; 798 + } 799 + 924 800 static bool xe_svm_range_is_valid(struct xe_svm_range *range, 925 801 struct xe_tile *tile, 926 - bool devmem_only) 802 + bool devmem_only, 803 + const struct drm_pagemap *dpagemap) 804 + 927 805 { 928 806 return (xe_vm_has_valid_gpu_mapping(tile, range->tile_present, 929 807 range->tile_invalidated) && 930 - (!devmem_only || xe_svm_range_in_vram(range))); 808 + (!devmem_only || xe_svm_range_has_pagemap(range, dpagemap))); 931 809 } 932 810 933 811 /** xe_svm_range_migrate_to_smem() - Move range pages from VRAM to SMEM ··· 969 805 * @vm: xe_vm pointer 970 806 * @range: Pointer to the SVM range structure 971 807 * @tile_mask: Mask representing the tiles to be checked 972 - * @devmem_preferred : if true range needs to be in devmem 808 + * @dpagemap: if !%NULL, the range is expected to be present 809 + * in device memory identified by this parameter. 973 810 * 974 811 * The xe_svm_range_validate() function checks if a range is 975 812 * valid and located in the desired memory region. ··· 979 814 */ 980 815 bool xe_svm_range_validate(struct xe_vm *vm, 981 816 struct xe_svm_range *range, 982 - u8 tile_mask, bool devmem_preferred) 817 + u8 tile_mask, const struct drm_pagemap *dpagemap) 983 818 { 984 819 bool ret; 985 820 986 821 xe_svm_notifier_lock(vm); 987 822 988 - ret = (range->tile_present & ~range->tile_invalidated & tile_mask) == tile_mask && 989 - (devmem_preferred == range->base.pages.flags.has_devmem_pages); 823 + ret = (range->tile_present & ~range->tile_invalidated & tile_mask) == tile_mask; 824 + if (dpagemap) 825 + ret = ret && xe_svm_range_has_pagemap_locked(range, dpagemap); 990 826 991 827 xe_svm_notifier_unlock(vm); 992 828 ··· 1022 856 struct mm_struct *mm, 1023 857 unsigned long timeslice_ms) 1024 858 { 1025 - struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); 859 + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); 860 + struct drm_pagemap_migrate_details mdetails = { 861 + .timeslice_ms = timeslice_ms, 862 + .source_peer_migrates = 1, 863 + }; 864 + struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); 865 + struct dma_fence *pre_migrate_fence = NULL; 1026 866 struct xe_device *xe = vr->xe; 1027 867 struct device *dev = xe->drm.dev; 1028 868 struct drm_buddy_block *block; ··· 1055 883 break; 1056 884 } 1057 885 886 + /* Ensure that any clearing or async eviction will complete before migration. */ 887 + if (!dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL)) { 888 + err = dma_resv_get_singleton(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 889 + &pre_migrate_fence); 890 + if (err) 891 + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 892 + false, MAX_SCHEDULE_TIMEOUT); 893 + else if (pre_migrate_fence) 894 + dma_fence_enable_sw_signaling(pre_migrate_fence); 895 + } 896 + 1058 897 drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, 1059 - &dpagemap_devmem_ops, dpagemap, end - start); 898 + &dpagemap_devmem_ops, dpagemap, end - start, 899 + pre_migrate_fence); 1060 900 1061 901 blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; 1062 902 list_for_each_entry(block, blocks, link) ··· 1078 894 1079 895 /* Ensure the device has a pm ref while there are device pages active. */ 1080 896 xe_pm_runtime_get_noresume(xe); 897 + /* Consumes the devmem allocation ref. */ 1081 898 err = drm_pagemap_migrate_to_devmem(&bo->devmem_allocation, mm, 1082 - start, end, timeslice_ms, 1083 - xe_svm_devm_owner(xe)); 1084 - if (err) 1085 - xe_svm_devmem_release(&bo->devmem_allocation); 899 + start, end, &mdetails); 1086 900 xe_bo_unlock(bo); 1087 901 xe_bo_put(bo); 1088 902 } ··· 1103 921 * xe_svm_range_needs_migrate_to_vram() - SVM range needs migrate to VRAM or not 1104 922 * @range: SVM range for which migration needs to be decided 1105 923 * @vma: vma which has range 1106 - * @preferred_region_is_vram: preferred region for range is vram 924 + * @dpagemap: The preferred struct drm_pagemap to migrate to. 1107 925 * 1108 926 * Return: True for range needing migration and migration is supported else false 1109 927 */ 1110 928 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 1111 - bool preferred_region_is_vram) 929 + const struct drm_pagemap *dpagemap) 1112 930 { 1113 931 struct xe_vm *vm = range_to_vm(&range->base); 1114 932 u64 range_size = xe_svm_range_size(range); 1115 933 1116 - if (!range->base.pages.flags.migrate_devmem || !preferred_region_is_vram) 934 + if (!range->base.pages.flags.migrate_devmem || !dpagemap) 1117 935 return false; 1118 936 1119 937 xe_assert(vm->xe, IS_DGFX(vm->xe)); 1120 938 1121 - if (xe_svm_range_in_vram(range)) { 1122 - drm_info(&vm->xe->drm, "Range is already in VRAM\n"); 939 + if (xe_svm_range_has_pagemap(range, dpagemap)) { 940 + drm_dbg(&vm->xe->drm, "Range is already in VRAM\n"); 1123 941 return false; 1124 942 } 1125 943 ··· 1193 1011 .devmem_only = need_vram && devmem_possible, 1194 1012 .timeslice_ms = need_vram && devmem_possible ? 1195 1013 vm->xe->atomic_svm_timeslice_ms : 0, 1196 - .device_private_page_owner = xe_svm_devm_owner(vm->xe), 1197 1014 }; 1198 1015 struct xe_validation_ctx vctx; 1199 1016 struct drm_exec exec; ··· 1215 1034 if (err) 1216 1035 return err; 1217 1036 1218 - dpagemap = xe_vma_resolve_pagemap(vma, tile); 1219 - if (!dpagemap && !ctx.devmem_only) 1220 - ctx.device_private_page_owner = NULL; 1037 + dpagemap = ctx.devmem_only ? xe_tile_local_pagemap(tile) : 1038 + xe_vma_resolve_pagemap(vma, tile); 1039 + ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); 1221 1040 range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx); 1222 1041 1223 1042 if (IS_ERR(range)) ··· 1230 1049 goto out; 1231 1050 } 1232 1051 1233 - if (xe_svm_range_is_valid(range, tile, ctx.devmem_only)) { 1052 + if (xe_svm_range_is_valid(range, tile, ctx.devmem_only, dpagemap)) { 1234 1053 xe_svm_range_valid_fault_count_stats_incr(gt, range); 1235 1054 range_debug(range, "PAGE FAULT - VALID"); 1236 1055 goto out; ··· 1239 1058 range_debug(range, "PAGE FAULT"); 1240 1059 1241 1060 if (--migrate_try_count >= 0 && 1242 - xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap || ctx.devmem_only)) { 1061 + xe_svm_range_needs_migrate_to_vram(range, vma, dpagemap)) { 1243 1062 ktime_t migrate_start = xe_gt_stats_ktime_get(); 1244 1063 1245 - /* TODO : For multi-device dpagemap will be used to find the 1246 - * remote tile and remote device. Will need to modify 1247 - * xe_svm_alloc_vram to use dpagemap for future multi-device 1248 - * support. 1249 - */ 1250 1064 xe_svm_range_migrate_count_stats_incr(gt, range); 1251 - err = xe_svm_alloc_vram(tile, range, &ctx); 1065 + err = xe_svm_alloc_vram(range, &ctx, dpagemap); 1252 1066 xe_svm_range_migrate_us_stats_incr(gt, range, migrate_start); 1253 1067 ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ 1254 1068 if (err) { ··· 1294 1118 if (err) { 1295 1119 range_debug(range, "PAGE FAULT - FAIL PAGE COLLECT"); 1296 1120 goto out; 1121 + } else if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) { 1122 + drm_dbg(&vm->xe->drm, "After page collect data location is %sin \"%s\".\n", 1123 + xe_svm_range_has_pagemap(range, dpagemap) ? "" : "NOT ", 1124 + dpagemap ? dpagemap->drm->unique : "System."); 1297 1125 } 1298 1126 1299 1127 xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start); ··· 1545 1365 1546 1366 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 1547 1367 1548 - static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile) 1549 - { 1550 - return &tile->mem.vram->dpagemap; 1551 - } 1552 - 1553 1368 /** 1554 1369 * xe_vma_resolve_pagemap - Resolve the appropriate DRM pagemap for a VMA 1555 1370 * @vma: Pointer to the xe_vma structure containing memory attributes ··· 1564 1389 */ 1565 1390 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 1566 1391 { 1567 - s32 fd = (s32)vma->attr.preferred_loc.devmem_fd; 1392 + struct drm_pagemap *dpagemap = vma->attr.preferred_loc.dpagemap; 1393 + s32 fd; 1394 + 1395 + if (dpagemap) 1396 + return dpagemap; 1397 + 1398 + fd = (s32)vma->attr.preferred_loc.devmem_fd; 1568 1399 1569 1400 if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM) 1570 1401 return NULL; 1571 1402 1572 1403 if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE) 1573 - return IS_DGFX(tile_to_xe(tile)) ? tile_local_pagemap(tile) : NULL; 1404 + return IS_DGFX(tile_to_xe(tile)) ? xe_tile_local_pagemap(tile) : NULL; 1574 1405 1575 - /* TODO: Support multi-device with drm_pagemap_from_fd(fd) */ 1576 1406 return NULL; 1577 1407 } 1578 1408 1579 1409 /** 1580 1410 * xe_svm_alloc_vram()- Allocate device memory pages for range, 1581 1411 * migrating existing data. 1582 - * @tile: tile to allocate vram from 1583 1412 * @range: SVM range 1584 1413 * @ctx: DRM GPU SVM context 1414 + * @dpagemap: The struct drm_pagemap representing the memory to allocate. 1585 1415 * 1586 1416 * Return: 0 on success, error code on failure. 1587 1417 */ 1588 - int xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, 1589 - const struct drm_gpusvm_ctx *ctx) 1418 + int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 1419 + struct drm_pagemap *dpagemap) 1590 1420 { 1591 - struct drm_pagemap *dpagemap; 1421 + static DECLARE_RWSEM(driver_migrate_lock); 1422 + struct xe_vm *vm = range_to_vm(&range->base); 1423 + enum drm_gpusvm_scan_result migration_state; 1424 + struct xe_device *xe = vm->xe; 1425 + int err, retries = 1; 1426 + bool write_locked = false; 1592 1427 1593 - xe_assert(tile_to_xe(tile), range->base.pages.flags.migrate_devmem); 1428 + xe_assert(range_to_vm(&range->base)->xe, range->base.pages.flags.migrate_devmem); 1594 1429 range_debug(range, "ALLOCATE VRAM"); 1595 1430 1596 - dpagemap = tile_local_pagemap(tile); 1597 - return drm_pagemap_populate_mm(dpagemap, xe_svm_range_start(range), 1598 - xe_svm_range_end(range), 1599 - range->base.gpusvm->mm, 1600 - ctx->timeslice_ms); 1431 + migration_state = drm_gpusvm_scan_mm(&range->base, 1432 + xe_svm_private_page_owner(vm, false), 1433 + dpagemap->pagemap); 1434 + 1435 + if (migration_state == DRM_GPUSVM_SCAN_EQUAL) { 1436 + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) 1437 + drm_dbg(dpagemap->drm, "Already migrated!\n"); 1438 + return 0; 1439 + } 1440 + 1441 + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) 1442 + drm_dbg(&xe->drm, "Request migration to device memory on \"%s\".\n", 1443 + dpagemap->drm->unique); 1444 + 1445 + err = down_read_interruptible(&driver_migrate_lock); 1446 + if (err) 1447 + return err; 1448 + do { 1449 + err = drm_pagemap_populate_mm(dpagemap, xe_svm_range_start(range), 1450 + xe_svm_range_end(range), 1451 + range->base.gpusvm->mm, 1452 + ctx->timeslice_ms); 1453 + 1454 + if (err == -EBUSY && retries) { 1455 + if (!write_locked) { 1456 + int lock_err; 1457 + 1458 + up_read(&driver_migrate_lock); 1459 + lock_err = down_write_killable(&driver_migrate_lock); 1460 + if (lock_err) 1461 + return lock_err; 1462 + write_locked = true; 1463 + } 1464 + drm_gpusvm_range_evict(range->base.gpusvm, &range->base); 1465 + } 1466 + } while (err == -EBUSY && retries--); 1467 + if (write_locked) 1468 + up_write(&driver_migrate_lock); 1469 + else 1470 + up_read(&driver_migrate_lock); 1471 + 1472 + return err; 1601 1473 } 1602 1474 1603 1475 static struct drm_pagemap_addr ··· 1654 1432 unsigned int order, 1655 1433 enum dma_data_direction dir) 1656 1434 { 1657 - struct device *pgmap_dev = dpagemap->dev; 1435 + struct device *pgmap_dev = dpagemap->drm->dev; 1658 1436 enum drm_interconnect_protocol prot; 1659 1437 dma_addr_t addr; 1660 1438 1661 1439 if (pgmap_dev == dev) { 1662 - addr = xe_vram_region_page_to_dpa(page_to_vr(page), page); 1440 + addr = xe_page_to_dpa(page); 1663 1441 prot = XE_INTERCONNECT_VRAM; 1664 1442 } else { 1665 - addr = DMA_MAPPING_ERROR; 1666 - prot = 0; 1443 + addr = dma_map_resource(dev, 1444 + xe_page_to_pcie(page), 1445 + PAGE_SIZE << order, dir, 1446 + DMA_ATTR_SKIP_CPU_SYNC); 1447 + prot = XE_INTERCONNECT_P2P; 1667 1448 } 1668 1449 1669 1450 return drm_pagemap_addr_encode(addr, prot, order, dir); 1670 1451 } 1671 1452 1453 + static void xe_drm_pagemap_device_unmap(struct drm_pagemap *dpagemap, 1454 + struct device *dev, 1455 + struct drm_pagemap_addr addr) 1456 + { 1457 + if (addr.proto != XE_INTERCONNECT_P2P) 1458 + return; 1459 + 1460 + dma_unmap_resource(dev, addr.addr, PAGE_SIZE << addr.order, 1461 + addr.dir, DMA_ATTR_SKIP_CPU_SYNC); 1462 + } 1463 + 1464 + static void xe_pagemap_destroy_work(struct work_struct *work) 1465 + { 1466 + struct xe_pagemap *xpagemap = container_of(work, typeof(*xpagemap), destroy_work); 1467 + struct dev_pagemap *pagemap = &xpagemap->pagemap; 1468 + struct drm_device *drm = xpagemap->dpagemap.drm; 1469 + int idx; 1470 + 1471 + /* 1472 + * Only unmap / release if devm_ release hasn't run yet. 1473 + * Otherwise the devm_ callbacks have already released, or 1474 + * will do shortly. 1475 + */ 1476 + if (drm_dev_enter(drm, &idx)) { 1477 + devm_memunmap_pages(drm->dev, pagemap); 1478 + devm_release_mem_region(drm->dev, pagemap->range.start, 1479 + pagemap->range.end - pagemap->range.start + 1); 1480 + drm_dev_exit(idx); 1481 + } 1482 + 1483 + drm_pagemap_release_owner(&xpagemap->peer); 1484 + kfree(xpagemap); 1485 + } 1486 + 1487 + static void xe_pagemap_destroy(struct drm_pagemap *dpagemap, bool from_atomic_or_reclaim) 1488 + { 1489 + struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); 1490 + struct xe_device *xe = to_xe_device(dpagemap->drm); 1491 + 1492 + if (from_atomic_or_reclaim) 1493 + queue_work(xe->destroy_wq, &xpagemap->destroy_work); 1494 + else 1495 + xe_pagemap_destroy_work(&xpagemap->destroy_work); 1496 + } 1497 + 1672 1498 static const struct drm_pagemap_ops xe_drm_pagemap_ops = { 1673 1499 .device_map = xe_drm_pagemap_device_map, 1500 + .device_unmap = xe_drm_pagemap_device_unmap, 1674 1501 .populate_mm = xe_drm_pagemap_populate_mm, 1502 + .destroy = xe_pagemap_destroy, 1675 1503 }; 1676 1504 1677 1505 /** 1678 - * xe_devm_add: Remap and provide memmap backing for device memory 1679 - * @tile: tile that the memory region belongs to 1680 - * @vr: vram memory region to remap 1506 + * xe_pagemap_create() - Create a struct xe_pagemap object 1507 + * @xe: The xe device. 1508 + * @vr: Back-pointer to the struct xe_vram_region. 1681 1509 * 1682 - * This remap device memory to host physical address space and create 1683 - * struct page to back device memory 1510 + * Allocate and initialize a struct xe_pagemap. On successful 1511 + * return, drm_pagemap_put() on the embedded struct drm_pagemap 1512 + * should be used to unreference. 1684 1513 * 1685 - * Return: 0 on success standard error code otherwise 1514 + * Return: Pointer to a struct xe_pagemap if successful. Error pointer 1515 + * on failure. 1686 1516 */ 1687 - int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 1517 + static struct xe_pagemap *xe_pagemap_create(struct xe_device *xe, struct xe_vram_region *vr) 1688 1518 { 1689 - struct xe_device *xe = tile_to_xe(tile); 1690 - struct device *dev = &to_pci_dev(xe->drm.dev)->dev; 1519 + struct device *dev = xe->drm.dev; 1520 + struct xe_pagemap *xpagemap; 1521 + struct dev_pagemap *pagemap; 1522 + struct drm_pagemap *dpagemap; 1691 1523 struct resource *res; 1692 1524 void *addr; 1693 - int ret; 1525 + int err; 1526 + 1527 + xpagemap = kzalloc(sizeof(*xpagemap), GFP_KERNEL); 1528 + if (!xpagemap) 1529 + return ERR_PTR(-ENOMEM); 1530 + 1531 + pagemap = &xpagemap->pagemap; 1532 + dpagemap = &xpagemap->dpagemap; 1533 + INIT_WORK(&xpagemap->destroy_work, xe_pagemap_destroy_work); 1534 + xpagemap->vr = vr; 1535 + xpagemap->peer.private = XE_PEER_PAGEMAP; 1536 + 1537 + err = drm_pagemap_init(dpagemap, pagemap, &xe->drm, &xe_drm_pagemap_ops); 1538 + if (err) 1539 + goto out_no_dpagemap; 1694 1540 1695 1541 res = devm_request_free_mem_region(dev, &iomem_resource, 1696 1542 vr->usable_size); 1697 1543 if (IS_ERR(res)) { 1698 - ret = PTR_ERR(res); 1699 - return ret; 1544 + err = PTR_ERR(res); 1545 + goto out_err; 1700 1546 } 1701 1547 1702 - vr->pagemap.type = MEMORY_DEVICE_PRIVATE; 1703 - vr->pagemap.range.start = res->start; 1704 - vr->pagemap.range.end = res->end; 1705 - vr->pagemap.nr_range = 1; 1706 - vr->pagemap.ops = drm_pagemap_pagemap_ops_get(); 1707 - vr->pagemap.owner = xe_svm_devm_owner(xe); 1708 - addr = devm_memremap_pages(dev, &vr->pagemap); 1548 + err = drm_pagemap_acquire_owner(&xpagemap->peer, &xe_owner_list, 1549 + xe_has_interconnect); 1550 + if (err) 1551 + goto out_no_owner; 1709 1552 1710 - vr->dpagemap.dev = dev; 1711 - vr->dpagemap.ops = &xe_drm_pagemap_ops; 1712 - 1553 + pagemap->type = MEMORY_DEVICE_PRIVATE; 1554 + pagemap->range.start = res->start; 1555 + pagemap->range.end = res->end; 1556 + pagemap->nr_range = 1; 1557 + pagemap->owner = xpagemap->peer.owner; 1558 + pagemap->ops = drm_pagemap_pagemap_ops_get(); 1559 + addr = devm_memremap_pages(dev, pagemap); 1713 1560 if (IS_ERR(addr)) { 1714 - devm_release_mem_region(dev, res->start, resource_size(res)); 1715 - ret = PTR_ERR(addr); 1716 - drm_err(&xe->drm, "Failed to remap tile %d memory, errno %pe\n", 1717 - tile->id, ERR_PTR(ret)); 1718 - return ret; 1561 + err = PTR_ERR(addr); 1562 + goto out_no_pages; 1719 1563 } 1720 - vr->hpa_base = res->start; 1564 + xpagemap->hpa_base = res->start; 1565 + return xpagemap; 1721 1566 1722 - drm_dbg(&xe->drm, "Added tile %d memory [%llx-%llx] to devm, remapped to %pr\n", 1723 - tile->id, vr->io_start, vr->io_start + vr->usable_size, res); 1567 + out_no_pages: 1568 + drm_pagemap_release_owner(&xpagemap->peer); 1569 + out_no_owner: 1570 + devm_release_mem_region(dev, res->start, res->end - res->start + 1); 1571 + out_err: 1572 + drm_pagemap_put(dpagemap); 1573 + return ERR_PTR(err); 1574 + 1575 + out_no_dpagemap: 1576 + kfree(xpagemap); 1577 + return ERR_PTR(err); 1578 + } 1579 + 1580 + /** 1581 + * xe_pagemap_find_or_create() - Find or create a struct xe_pagemap 1582 + * @xe: The xe device. 1583 + * @cache: The struct xe_pagemap_cache. 1584 + * @vr: The VRAM region. 1585 + * 1586 + * Check if there is an already used xe_pagemap for this tile, and in that case, 1587 + * return it. 1588 + * If not, check if there is a cached xe_pagemap for this tile, and in that case, 1589 + * cancel its destruction, re-initialize it and return it. 1590 + * Finally if there is no cached or already used pagemap, create one and 1591 + * register it in the tile's pagemap cache. 1592 + * 1593 + * Note that this function is typically called from within an IOCTL, and waits are 1594 + * therefore carried out interruptible if possible. 1595 + * 1596 + * Return: A pointer to a struct xe_pagemap if successful, Error pointer on failure. 1597 + */ 1598 + static struct xe_pagemap * 1599 + xe_pagemap_find_or_create(struct xe_device *xe, struct drm_pagemap_cache *cache, 1600 + struct xe_vram_region *vr) 1601 + { 1602 + struct drm_pagemap *dpagemap; 1603 + struct xe_pagemap *xpagemap; 1604 + int err; 1605 + 1606 + err = drm_pagemap_cache_lock_lookup(cache); 1607 + if (err) 1608 + return ERR_PTR(err); 1609 + 1610 + dpagemap = drm_pagemap_get_from_cache(cache); 1611 + if (IS_ERR(dpagemap)) { 1612 + xpagemap = ERR_CAST(dpagemap); 1613 + } else if (!dpagemap) { 1614 + xpagemap = xe_pagemap_create(xe, vr); 1615 + if (IS_ERR(xpagemap)) 1616 + goto out_unlock; 1617 + drm_pagemap_cache_set_pagemap(cache, &xpagemap->dpagemap); 1618 + } else { 1619 + xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); 1620 + } 1621 + 1622 + out_unlock: 1623 + drm_pagemap_cache_unlock_lookup(cache); 1624 + return xpagemap; 1625 + } 1626 + 1627 + static int xe_svm_get_pagemaps(struct xe_vm *vm) 1628 + { 1629 + struct xe_device *xe = vm->xe; 1630 + struct xe_pagemap *xpagemap; 1631 + struct xe_tile *tile; 1632 + int id; 1633 + 1634 + for_each_tile(tile, xe, id) { 1635 + struct xe_vram_region *vr; 1636 + 1637 + if (!((BIT(id) << 1) & xe->info.mem_region_mask)) 1638 + continue; 1639 + 1640 + vr = xe_tile_to_vr(tile); 1641 + xpagemap = xe_pagemap_find_or_create(xe, vr->dpagemap_cache, vr); 1642 + if (IS_ERR(xpagemap)) 1643 + break; 1644 + vm->svm.pagemaps[id] = xpagemap; 1645 + } 1646 + 1647 + if (IS_ERR(xpagemap)) { 1648 + xe_svm_put_pagemaps(vm); 1649 + return PTR_ERR(xpagemap); 1650 + } 1651 + 1724 1652 return 0; 1725 1653 } 1654 + 1655 + /** 1656 + * xe_pagemap_shrinker_create() - Create a drm_pagemap shrinker 1657 + * @xe: The xe device 1658 + * 1659 + * Create a drm_pagemap shrinker and register with the xe device. 1660 + * 1661 + * Return: %0 on success, negative error code on failure. 1662 + */ 1663 + int xe_pagemap_shrinker_create(struct xe_device *xe) 1664 + { 1665 + xe->usm.dpagemap_shrinker = drm_pagemap_shrinker_create_devm(&xe->drm); 1666 + return PTR_ERR_OR_ZERO(xe->usm.dpagemap_shrinker); 1667 + } 1668 + 1669 + /** 1670 + * xe_pagemap_cache_create() - Create a drm_pagemap cache 1671 + * @tile: The tile to register the cache with 1672 + * 1673 + * Create a drm_pagemap cache and register with the tile. 1674 + * 1675 + * Return: %0 on success, negative error code on failure. 1676 + */ 1677 + int xe_pagemap_cache_create(struct xe_tile *tile) 1678 + { 1679 + struct xe_device *xe = tile_to_xe(tile); 1680 + 1681 + if (IS_DGFX(xe)) { 1682 + struct drm_pagemap_cache *cache = 1683 + drm_pagemap_cache_create_devm(xe->usm.dpagemap_shrinker); 1684 + 1685 + if (IS_ERR(cache)) 1686 + return PTR_ERR(cache); 1687 + 1688 + tile->mem.vram->dpagemap_cache = cache; 1689 + } 1690 + 1691 + return 0; 1692 + } 1693 + 1694 + static struct drm_pagemap *xe_devmem_open(struct xe_device *xe, u32 region_instance) 1695 + { 1696 + u32 tile_id = region_instance - 1; 1697 + struct xe_pagemap *xpagemap; 1698 + struct xe_vram_region *vr; 1699 + 1700 + if (tile_id >= xe->info.tile_count) 1701 + return ERR_PTR(-ENOENT); 1702 + 1703 + if (!((BIT(tile_id) << 1) & xe->info.mem_region_mask)) 1704 + return ERR_PTR(-ENOENT); 1705 + 1706 + vr = xe_tile_to_vr(&xe->tiles[tile_id]); 1707 + 1708 + /* Returns a reference-counted embedded struct drm_pagemap */ 1709 + xpagemap = xe_pagemap_find_or_create(xe, vr->dpagemap_cache, vr); 1710 + if (IS_ERR(xpagemap)) 1711 + return ERR_CAST(xpagemap); 1712 + 1713 + return &xpagemap->dpagemap; 1714 + } 1715 + 1716 + /** 1717 + * xe_drm_pagemap_from_fd() - Return a drm_pagemap pointer from a 1718 + * (file_descriptor, region_instance) pair. 1719 + * @fd: An fd opened against an xe device. 1720 + * @region_instance: The region instance representing the device memory 1721 + * on the opened xe device. 1722 + * 1723 + * Opens a struct drm_pagemap pointer on the 1724 + * indicated device and region_instance. 1725 + * 1726 + * Return: A reference-counted struct drm_pagemap pointer on success, 1727 + * negative error pointer on failure. 1728 + */ 1729 + struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) 1730 + { 1731 + struct drm_pagemap *dpagemap; 1732 + struct file *file; 1733 + struct drm_file *fpriv; 1734 + struct drm_device *drm; 1735 + int idx; 1736 + 1737 + if (fd <= 0) 1738 + return ERR_PTR(-EINVAL); 1739 + 1740 + file = fget(fd); 1741 + if (!file) 1742 + return ERR_PTR(-ENOENT); 1743 + 1744 + if (!xe_is_xe_file(file)) { 1745 + dpagemap = ERR_PTR(-ENOENT); 1746 + goto out; 1747 + } 1748 + 1749 + fpriv = file->private_data; 1750 + drm = fpriv->minor->dev; 1751 + if (!drm_dev_enter(drm, &idx)) { 1752 + dpagemap = ERR_PTR(-ENODEV); 1753 + goto out; 1754 + } 1755 + 1756 + dpagemap = xe_devmem_open(to_xe_device(drm), region_instance); 1757 + drm_dev_exit(idx); 1758 + out: 1759 + fput(file); 1760 + return dpagemap; 1761 + } 1762 + 1726 1763 #else 1727 - int xe_svm_alloc_vram(struct xe_tile *tile, 1728 - struct xe_svm_range *range, 1729 - const struct drm_gpusvm_ctx *ctx) 1764 + 1765 + int xe_pagemap_shrinker_create(struct xe_device *xe) 1766 + { 1767 + return 0; 1768 + } 1769 + 1770 + int xe_pagemap_cache_create(struct xe_tile *tile) 1771 + { 1772 + return 0; 1773 + } 1774 + 1775 + int xe_svm_alloc_vram(struct xe_svm_range *range, 1776 + const struct drm_gpusvm_ctx *ctx, 1777 + struct drm_pagemap *dpagemap) 1730 1778 { 1731 1779 return -EOPNOTSUPP; 1732 - } 1733 - 1734 - int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 1735 - { 1736 - return 0; 1737 1780 } 1738 1781 1739 1782 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 1740 1783 { 1741 1784 return NULL; 1742 1785 } 1786 + 1787 + struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) 1788 + { 1789 + return ERR_PTR(-ENOENT); 1790 + } 1791 + 1743 1792 #endif 1744 1793 1745 1794 /**

+63 -22

drivers/gpu/drm/xe/xe_svm.h

··· 6 6 #ifndef _XE_SVM_H_ 7 7 #define _XE_SVM_H_ 8 8 9 - struct xe_device; 10 - 11 - /** 12 - * xe_svm_devm_owner() - Return the owner of device private memory 13 - * @xe: The xe device. 14 - * 15 - * Return: The owner of this device's device private memory to use in 16 - * hmm_range_fault()- 17 - */ 18 - static inline void *xe_svm_devm_owner(struct xe_device *xe) 19 - { 20 - return xe; 21 - } 22 - 23 9 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM) 24 10 25 11 #include <drm/drm_pagemap.h> 26 12 #include <drm/drm_gpusvm.h> 13 + #include <drm/drm_pagemap_util.h> 27 14 28 15 #define XE_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER 16 + #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 17 + 18 + struct drm_device; 19 + struct drm_file; 29 20 30 21 struct xe_bo; 31 22 struct xe_gt; 23 + struct xe_device; 24 + struct xe_vram_region; 32 25 struct xe_tile; 33 26 struct xe_vm; 34 27 struct xe_vma; ··· 46 53 * range. Protected by GPU SVM notifier lock. 47 54 */ 48 55 u8 tile_invalidated; 56 + }; 57 + 58 + /** 59 + * struct xe_pagemap - Manages xe device_private memory for SVM. 60 + * @pagemap: The struct dev_pagemap providing the struct pages. 61 + * @dpagemap: The drm_pagemap managing allocation and migration. 62 + * @destroy_work: Handles asnynchronous destruction and caching. 63 + * @peer: Used for pagemap owner computation. 64 + * @hpa_base: The host physical address base for the managemd memory. 65 + * @vr: Backpointer to the xe_vram region. 66 + */ 67 + struct xe_pagemap { 68 + struct dev_pagemap pagemap; 69 + struct drm_pagemap dpagemap; 70 + struct work_struct destroy_work; 71 + struct drm_pagemap_peer peer; 72 + resource_size_t hpa_base; 73 + struct xe_vram_region *vr; 49 74 }; 50 75 51 76 /** ··· 95 84 96 85 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation); 97 86 98 - int xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, 99 - const struct drm_gpusvm_ctx *ctx); 87 + int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 88 + struct drm_pagemap *dpagemap); 100 89 101 90 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 102 91 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx); ··· 105 94 struct drm_gpusvm_ctx *ctx); 106 95 107 96 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 108 - bool preferred_region_is_vram); 97 + const struct drm_pagemap *dpagemap); 109 98 110 99 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range); 111 100 112 101 bool xe_svm_range_validate(struct xe_vm *vm, 113 102 struct xe_svm_range *range, 114 - u8 tile_mask, bool devmem_preferred); 103 + u8 tile_mask, const struct drm_pagemap *dpagemap); 115 104 116 105 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end, struct xe_vma *vma); 117 106 ··· 120 109 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end); 121 110 122 111 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile); 112 + 113 + void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem); 123 114 124 115 /** 125 116 * xe_svm_range_has_dma_mapping() - SVM range has DMA mapping ··· 184 171 185 172 void xe_svm_flush(struct xe_vm *vm); 186 173 174 + int xe_pagemap_shrinker_create(struct xe_device *xe); 175 + 176 + int xe_pagemap_cache_create(struct xe_tile *tile); 177 + 178 + struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance); 179 + 187 180 #else 188 181 #include <linux/interval_tree.h> 189 182 #include "xe_vm.h" ··· 198 179 struct drm_gpusvm_ctx; 199 180 struct drm_gpusvm_range; 200 181 struct xe_bo; 201 - struct xe_gt; 182 + struct xe_device; 202 183 struct xe_vm; 203 184 struct xe_vma; 204 185 struct xe_tile; 205 186 struct xe_vram_region; 206 187 207 188 #define XE_INTERCONNECT_VRAM 1 189 + #define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1) 208 190 209 191 struct xe_svm_range { 210 192 struct { ··· 280 260 } 281 261 282 262 static inline int 283 - xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, 284 - const struct drm_gpusvm_ctx *ctx) 263 + xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *ctx, 264 + struct drm_pagemap *dpagemap) 285 265 { 286 266 return -EOPNOTSUPP; 287 267 } ··· 322 302 323 303 static inline 324 304 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 325 - u32 region) 305 + const struct drm_pagemap *dpagemap) 326 306 { 327 307 return false; 328 308 } ··· 363 343 return NULL; 364 344 } 365 345 346 + static inline void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) 347 + { 348 + return NULL; 349 + } 350 + 366 351 static inline void xe_svm_flush(struct xe_vm *vm) 367 352 { 368 353 } 354 + 355 + static inline int xe_pagemap_shrinker_create(struct xe_device *xe) 356 + { 357 + return 0; 358 + } 359 + 360 + static inline int xe_pagemap_cache_create(struct xe_tile *tile) 361 + { 362 + return 0; 363 + } 364 + 365 + static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_instance) 366 + { 367 + return ERR_PTR(-ENOENT); 368 + } 369 + 369 370 #define xe_svm_range_has_dma_mapping(...) false 370 371 #endif /* CONFIG_DRM_XE_GPUSVM */ 371 372

+30 -4

drivers/gpu/drm/xe/xe_tile.c

··· 6 6 #include <linux/fault-inject.h> 7 7 8 8 #include <drm/drm_managed.h> 9 + #include <drm/drm_pagemap_util.h> 9 10 10 11 #include "xe_bo.h" 11 12 #include "xe_device.h" ··· 181 180 int xe_tile_init_noalloc(struct xe_tile *tile) 182 181 { 183 182 struct xe_device *xe = tile_to_xe(tile); 183 + int err; 184 184 185 185 xe_wa_apply_tile_workarounds(tile); 186 186 187 - if (xe->info.has_usm && IS_DGFX(xe)) 188 - xe_devm_add(tile, tile->mem.vram); 187 + err = xe_pagemap_cache_create(tile); 188 + if (err) 189 + return err; 189 190 190 191 if (IS_DGFX(xe) && !ttm_resource_manager_used(&tile->mem.vram->ttm.manager)) { 191 - int err = xe_ttm_vram_mgr_init(xe, tile->mem.vram); 192 - 192 + err = xe_ttm_vram_mgr_init(xe, tile->mem.vram); 193 193 if (err) 194 194 return err; 195 + 195 196 xe->info.mem_region_mask |= BIT(tile->mem.vram->id) << 1; 196 197 } 197 198 ··· 223 220 { 224 221 xe_migrate_wait(tile->migrate); 225 222 } 223 + 224 + #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 225 + /** 226 + * xe_tile_local_pagemap() - Return a pointer to the tile's local drm_pagemap if any 227 + * @tile: The tile. 228 + * 229 + * Return: A pointer to the tile's local drm_pagemap, or NULL if local pagemap 230 + * support has been compiled out. 231 + */ 232 + struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) 233 + { 234 + struct drm_pagemap *dpagemap = 235 + drm_pagemap_get_from_cache_if_active(xe_tile_to_vr(tile)->dpagemap_cache); 236 + 237 + if (dpagemap) { 238 + xe_assert(tile_to_xe(tile), kref_read(&dpagemap->ref) >= 2); 239 + drm_pagemap_put(dpagemap); 240 + } 241 + 242 + return dpagemap; 243 + } 244 + #endif 245 +

+21

drivers/gpu/drm/xe/xe_tile.h

··· 8 8 9 9 #include "xe_device_types.h" 10 10 11 + struct xe_pagemap; 11 12 struct xe_tile; 12 13 13 14 int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id); ··· 24 23 return tile->id == 0; 25 24 } 26 25 26 + /** 27 + * xe_tile_to_vr() - Return the struct xe_vram_region pointer from a 28 + * struct xe_tile pointer 29 + * @tile: Pointer to the struct xe_tile. 30 + * 31 + * Return: Pointer to the struct xe_vram_region embedded in *@tile. 32 + */ 33 + static inline struct xe_vram_region *xe_tile_to_vr(struct xe_tile *tile) 34 + { 35 + return tile->mem.vram; 36 + } 37 + 38 + #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 39 + struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile); 40 + #else 41 + static inline struct drm_pagemap *xe_tile_local_pagemap(struct xe_tile *tile) 42 + { 43 + return NULL; 44 + } 45 + #endif 27 46 #endif

+1 -1

drivers/gpu/drm/xe/xe_userptr.c

··· 55 55 struct xe_device *xe = vm->xe; 56 56 struct drm_gpusvm_ctx ctx = { 57 57 .read_only = xe_vma_read_only(vma), 58 - .device_private_page_owner = xe_svm_devm_owner(xe), 58 + .device_private_page_owner = xe_svm_private_page_owner(vm, false), 59 59 .allow_mixed = true, 60 60 }; 61 61

+45 -20

drivers/gpu/drm/xe/xe_vm.c

··· 957 957 return fence; 958 958 } 959 959 960 + static void xe_vma_mem_attr_fini(struct xe_vma_mem_attr *attr) 961 + { 962 + drm_pagemap_put(attr->preferred_loc.dpagemap); 963 + } 964 + 960 965 static void xe_vma_free(struct xe_vma *vma) 961 966 { 967 + xe_vma_mem_attr_fini(&vma->attr); 968 + 962 969 if (xe_vma_is_userptr(vma)) 963 970 kfree(to_userptr_vma(vma)); 964 971 else 965 972 kfree(vma); 973 + } 974 + 975 + /** 976 + * xe_vma_mem_attr_copy() - copy an xe_vma_mem_attr structure. 977 + * @to: Destination. 978 + * @from: Source. 979 + * 980 + * Copies an xe_vma_mem_attr structure taking care to get reference 981 + * counting of individual members right. 982 + */ 983 + void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from) 984 + { 985 + xe_vma_mem_attr_fini(to); 986 + *to = *from; 987 + if (to->preferred_loc.dpagemap) 988 + drm_pagemap_get(to->preferred_loc.dpagemap); 966 989 } 967 990 968 991 static struct xe_vma *xe_vma_create(struct xe_vm *vm, ··· 1038 1015 if (vm->xe->info.has_atomic_enable_pte_bit) 1039 1016 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; 1040 1017 1041 - vma->attr = *attr; 1042 - 1018 + xe_vma_mem_attr_copy(&vma->attr, attr); 1043 1019 if (bo) { 1044 1020 struct drm_gpuvm_bo *vm_bo; 1045 1021 ··· 2342 2320 struct xe_tile *tile; 2343 2321 struct xe_svm_range *svm_range; 2344 2322 struct drm_gpusvm_ctx ctx = {}; 2345 - struct drm_pagemap *dpagemap; 2323 + struct drm_pagemap *dpagemap = NULL; 2346 2324 u8 id, tile_mask = 0; 2347 2325 u32 i; 2348 2326 ··· 2360 2338 2361 2339 xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); 2362 2340 op->prefetch_range.ranges_count = 0; 2363 - tile = NULL; 2364 2341 2365 2342 if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { 2366 2343 dpagemap = xe_vma_resolve_pagemap(vma, 2367 2344 xe_device_get_root_tile(vm->xe)); 2368 - /* 2369 - * TODO: Once multigpu support is enabled will need 2370 - * something to dereference tile from dpagemap. 2371 - */ 2372 - if (dpagemap) 2373 - tile = xe_device_get_root_tile(vm->xe); 2374 2345 } else if (prefetch_region) { 2375 2346 tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - 2376 2347 XE_PL_VRAM0]; 2348 + dpagemap = xe_tile_local_pagemap(tile); 2377 2349 } 2378 2350 2379 - op->prefetch_range.tile = tile; 2351 + op->prefetch_range.dpagemap = dpagemap; 2380 2352 alloc_next_range: 2381 2353 svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx); 2382 2354 ··· 2389 2373 goto unwind_prefetch_ops; 2390 2374 } 2391 2375 2392 - if (xe_svm_range_validate(vm, svm_range, tile_mask, !!tile)) { 2376 + if (xe_svm_range_validate(vm, svm_range, tile_mask, dpagemap)) { 2393 2377 xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); 2394 2378 goto check_next_range; 2395 2379 } ··· 2911 2895 { 2912 2896 bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); 2913 2897 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); 2914 - struct xe_tile *tile = op->prefetch_range.tile; 2898 + struct drm_pagemap *dpagemap = op->prefetch_range.dpagemap; 2915 2899 int err = 0; 2916 2900 2917 2901 struct xe_svm_range *svm_range; ··· 2924 2908 ctx.read_only = xe_vma_read_only(vma); 2925 2909 ctx.devmem_possible = devmem_possible; 2926 2910 ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0; 2927 - ctx.device_private_page_owner = xe_svm_devm_owner(vm->xe); 2911 + ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); 2928 2912 2929 2913 /* TODO: Threading the migration */ 2930 2914 xa_for_each(&op->prefetch_range.range, i, svm_range) { 2931 - if (!tile) 2915 + if (!dpagemap) 2932 2916 xe_svm_range_migrate_to_smem(vm, svm_range); 2933 2917 2934 - if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, !!tile)) { 2935 - err = xe_svm_alloc_vram(tile, svm_range, &ctx); 2918 + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) { 2919 + drm_dbg(&vm->xe->drm, 2920 + "Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n", 2921 + dpagemap ? dpagemap->drm->unique : "system", 2922 + xe_svm_range_start(svm_range), xe_svm_range_end(svm_range)); 2923 + } 2924 + 2925 + if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) { 2926 + err = xe_svm_alloc_vram(svm_range, &ctx, dpagemap); 2936 2927 if (err) { 2937 2928 drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n", 2938 2929 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); ··· 4347 4324 struct drm_gpuva_op *__op; 4348 4325 unsigned int vma_flags = 0; 4349 4326 bool remap_op = false; 4350 - struct xe_vma_mem_attr tmp_attr; 4327 + struct xe_vma_mem_attr tmp_attr = {}; 4351 4328 u16 default_pat; 4352 4329 int err; 4353 4330 ··· 4442 4419 * VMA, so they can be assigned to newly MAP created vma. 4443 4420 */ 4444 4421 if (is_madvise) 4445 - tmp_attr = vma->attr; 4422 + xe_vma_mem_attr_copy(&tmp_attr, &vma->attr); 4446 4423 4447 4424 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); 4448 4425 } else if (__op->op == DRM_GPUVA_OP_MAP) { ··· 4452 4429 * copy them to new vma. 4453 4430 */ 4454 4431 if (is_madvise) 4455 - vma->attr = tmp_attr; 4432 + xe_vma_mem_attr_copy(&vma->attr, &tmp_attr); 4456 4433 } 4457 4434 } 4458 4435 4459 4436 xe_vm_unlock(vm); 4460 4437 drm_gpuva_ops_free(&vm->gpuvm, ops); 4438 + xe_vma_mem_attr_fini(&tmp_attr); 4461 4439 return 0; 4462 4440 4463 4441 unwind_ops: ··· 4556 4532 4557 4533 return xe_vm_alloc_vma(vm, &map_req, false); 4558 4534 } 4535 +

+1

drivers/gpu/drm/xe/xe_vm.h

··· 414 414 #define xe_vm_has_valid_gpu_mapping(tile, tile_present, tile_invalidated) \ 415 415 ((READ_ONCE(tile_present) & ~READ_ONCE(tile_invalidated)) & BIT((tile)->id)) 416 416 417 + void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from); 417 418 #endif

+86 -20

drivers/gpu/drm/xe/xe_vm_madvise.c

··· 22 22 bool has_svm_userptr_vmas; 23 23 }; 24 24 25 + /** 26 + * struct xe_madvise_details - Argument to madvise_funcs 27 + * @dpagemap: Reference-counted pointer to a struct drm_pagemap. 28 + * 29 + * The madvise IOCTL handler may, in addition to the user-space 30 + * args, have additional info to pass into the madvise_func that 31 + * handles the madvise type. Use a struct_xe_madvise_details 32 + * for that and extend the struct as necessary. 33 + */ 34 + struct xe_madvise_details { 35 + struct drm_pagemap *dpagemap; 36 + }; 37 + 25 38 static int get_vmas(struct xe_vm *vm, struct xe_vmas_in_madvise_range *madvise_range) 26 39 { 27 40 u64 addr = madvise_range->addr; ··· 87 74 88 75 static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, 89 76 struct xe_vma **vmas, int num_vmas, 90 - struct drm_xe_madvise *op) 77 + struct drm_xe_madvise *op, 78 + struct xe_madvise_details *details) 91 79 { 92 80 int i; 93 81 94 82 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC); 95 83 96 84 for (i = 0; i < num_vmas; i++) { 85 + struct xe_vma *vma = vmas[i]; 86 + struct xe_vma_preferred_loc *loc = &vma->attr.preferred_loc; 87 + 97 88 /*TODO: Extend attributes to bo based vmas */ 98 - if ((vmas[i]->attr.preferred_loc.devmem_fd == op->preferred_mem_loc.devmem_fd && 99 - vmas[i]->attr.preferred_loc.migration_policy == 100 - op->preferred_mem_loc.migration_policy) || 101 - !xe_vma_is_cpu_addr_mirror(vmas[i])) { 102 - vmas[i]->skip_invalidation = true; 89 + if ((loc->devmem_fd == op->preferred_mem_loc.devmem_fd && 90 + loc->migration_policy == op->preferred_mem_loc.migration_policy) || 91 + !xe_vma_is_cpu_addr_mirror(vma)) { 92 + vma->skip_invalidation = true; 103 93 } else { 104 - vmas[i]->skip_invalidation = false; 105 - vmas[i]->attr.preferred_loc.devmem_fd = op->preferred_mem_loc.devmem_fd; 94 + vma->skip_invalidation = false; 95 + loc->devmem_fd = op->preferred_mem_loc.devmem_fd; 106 96 /* Till multi-device support is not added migration_policy 107 97 * is of no use and can be ignored. 108 98 */ 109 - vmas[i]->attr.preferred_loc.migration_policy = 110 - op->preferred_mem_loc.migration_policy; 99 + loc->migration_policy = op->preferred_mem_loc.migration_policy; 100 + drm_pagemap_put(loc->dpagemap); 101 + loc->dpagemap = NULL; 102 + if (details->dpagemap) 103 + loc->dpagemap = drm_pagemap_get(details->dpagemap); 111 104 } 112 105 } 113 106 } 114 107 115 108 static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, 116 109 struct xe_vma **vmas, int num_vmas, 117 - struct drm_xe_madvise *op) 110 + struct drm_xe_madvise *op, 111 + struct xe_madvise_details *details) 118 112 { 119 113 struct xe_bo *bo; 120 114 int i; ··· 162 142 163 143 static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, 164 144 struct xe_vma **vmas, int num_vmas, 165 - struct drm_xe_madvise *op) 145 + struct drm_xe_madvise *op, 146 + struct xe_madvise_details *details) 166 147 { 167 148 int i; 168 149 ··· 181 160 182 161 typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, 183 162 struct xe_vma **vmas, int num_vmas, 184 - struct drm_xe_madvise *op); 163 + struct drm_xe_madvise *op, 164 + struct xe_madvise_details *details); 185 165 186 166 static const madvise_func madvise_funcs[] = { 187 167 [DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC] = madvise_preferred_mem_loc, ··· 266 244 if (XE_IOCTL_DBG(xe, fd < DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM)) 267 245 return false; 268 246 269 - if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > 270 - DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) 247 + if (XE_IOCTL_DBG(xe, fd <= DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && 248 + args->preferred_mem_loc.region_instance != 0)) 271 249 return false; 272 250 273 - if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.pad)) 251 + if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > 252 + DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) 274 253 return false; 275 254 276 255 if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.reserved)) ··· 315 292 return false; 316 293 317 294 return true; 295 + } 296 + 297 + static int xe_madvise_details_init(struct xe_vm *vm, const struct drm_xe_madvise *args, 298 + struct xe_madvise_details *details) 299 + { 300 + struct xe_device *xe = vm->xe; 301 + 302 + memset(details, 0, sizeof(*details)); 303 + 304 + if (args->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC) { 305 + int fd = args->preferred_mem_loc.devmem_fd; 306 + struct drm_pagemap *dpagemap; 307 + 308 + if (fd <= 0) 309 + return 0; 310 + 311 + dpagemap = xe_drm_pagemap_from_fd(args->preferred_mem_loc.devmem_fd, 312 + args->preferred_mem_loc.region_instance); 313 + if (XE_IOCTL_DBG(xe, IS_ERR(dpagemap))) 314 + return PTR_ERR(dpagemap); 315 + 316 + /* Don't allow a foreign placement without a fast interconnect! */ 317 + if (XE_IOCTL_DBG(xe, dpagemap->pagemap->owner != vm->svm.peer.owner)) { 318 + drm_pagemap_put(dpagemap); 319 + return -ENOLINK; 320 + } 321 + details->dpagemap = dpagemap; 322 + } 323 + 324 + return 0; 325 + } 326 + 327 + static void xe_madvise_details_fini(struct xe_madvise_details *details) 328 + { 329 + drm_pagemap_put(details->dpagemap); 318 330 } 319 331 320 332 static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, ··· 405 347 struct drm_xe_madvise *args = data; 406 348 struct xe_vmas_in_madvise_range madvise_range = {.addr = args->start, 407 349 .range = args->range, }; 350 + struct xe_madvise_details details; 408 351 struct xe_vm *vm; 409 352 struct drm_exec exec; 410 353 int err, attr_type; ··· 430 371 goto unlock_vm; 431 372 } 432 373 433 - err = xe_vm_alloc_madvise_vma(vm, args->start, args->range); 374 + err = xe_madvise_details_init(vm, args, &details); 434 375 if (err) 435 376 goto unlock_vm; 436 377 378 + err = xe_vm_alloc_madvise_vma(vm, args->start, args->range); 379 + if (err) 380 + goto madv_fini; 381 + 437 382 err = get_vmas(vm, &madvise_range); 438 383 if (err || !madvise_range.num_vmas) 439 - goto unlock_vm; 384 + goto madv_fini; 440 385 441 386 if (madvise_range.has_bo_vmas) { 442 387 if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { ··· 448 385 madvise_range.num_vmas, 449 386 args->atomic.val)) { 450 387 err = -EINVAL; 451 - goto unlock_vm; 388 + goto madv_fini; 452 389 } 453 390 } 454 391 ··· 474 411 } 475 412 476 413 attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs)); 477 - madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args); 414 + madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args, 415 + &details); 478 416 479 417 err = xe_vm_invalidate_madvise_range(vm, args->start, args->start + args->range); 480 418 ··· 487 423 drm_exec_fini(&exec); 488 424 kfree(madvise_range.vmas); 489 425 madvise_range.vmas = NULL; 426 + madv_fini: 427 + xe_madvise_details_fini(&details); 490 428 unlock_vm: 491 429 up_write(&vm->lock); 492 430 put_vm:

+17 -4

drivers/gpu/drm/xe/xe_vm_types.h

··· 8 8 9 9 #include <drm/drm_gpusvm.h> 10 10 #include <drm/drm_gpuvm.h> 11 + #include <drm/drm_pagemap_util.h> 11 12 12 13 #include <linux/dma-resv.h> 13 14 #include <linux/kref.h> ··· 19 18 #include "xe_pt_types.h" 20 19 #include "xe_range_fence.h" 21 20 #include "xe_userptr.h" 21 + 22 + struct drm_pagemap; 22 23 23 24 struct xe_bo; 24 25 struct xe_svm_range; ··· 56 53 */ 57 54 struct xe_vma_mem_attr { 58 55 /** @preferred_loc: preferred memory_location */ 59 - struct { 56 + struct xe_vma_preferred_loc { 60 57 /** @preferred_loc.migration_policy: Pages migration policy */ 61 58 u32 migration_policy; 62 59 ··· 67 64 * closest device memory respectively. 68 65 */ 69 66 u32 devmem_fd; 67 + /** 68 + * @preferred_loc.dpagemap: Reference-counted pointer to the drm_pagemap preferred 69 + * for migration on a SVM page-fault. The pointer is protected by the 70 + * vm lock, and is %NULL if @devmem_fd should be consulted for special 71 + * values. 72 + */ 73 + struct drm_pagemap *dpagemap; 70 74 } preferred_loc; 71 75 72 76 /** ··· 201 191 */ 202 192 struct work_struct work; 203 193 } garbage_collector; 194 + struct xe_pagemap *pagemaps[XE_MAX_TILES_PER_DEVICE]; 195 + /** @svm.peer: Used for pagemap connectivity computations. */ 196 + struct drm_pagemap_peer peer; 204 197 } svm; 205 198 206 199 struct xe_device *xe; ··· 408 395 /** @ranges_count: number of svm ranges to map */ 409 396 u32 ranges_count; 410 397 /** 411 - * @tile: Pointer to the tile structure containing memory to prefetch. 412 - * NULL if prefetch requested region is smem 398 + * @dpagemap: Pointer to the dpagemap structure containing memory to prefetch. 399 + * NULL if prefetch requested region is smem 413 400 */ 414 - struct xe_tile *tile; 401 + struct drm_pagemap *dpagemap; 415 402 }; 416 403 417 404 /** enum xe_vma_op_flags - flags for VMA operation */

-91

drivers/gpu/drm/xe/xe_vram.c

··· 25 25 #include "xe_vram.h" 26 26 #include "xe_vram_types.h" 27 27 28 - static void resize_bar(struct xe_device *xe, int resno, resource_size_t size) 29 - { 30 - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 31 - int bar_size = pci_rebar_bytes_to_size(size); 32 - int ret; 33 - 34 - ret = pci_resize_resource(pdev, resno, bar_size, 0); 35 - if (ret) { 36 - drm_info(&xe->drm, "Failed to resize BAR%d to %dM (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", 37 - resno, 1 << bar_size, ERR_PTR(ret)); 38 - return; 39 - } 40 - 41 - drm_info(&xe->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); 42 - } 43 - 44 - /* 45 - * if force_vram_bar_size is set, attempt to set to the requested size 46 - * else set to maximum possible size 47 - */ 48 - void xe_vram_resize_bar(struct xe_device *xe) 49 - { 50 - int force_vram_bar_size = xe_modparam.force_vram_bar_size; 51 - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 52 - struct pci_bus *root = pdev->bus; 53 - resource_size_t current_size; 54 - resource_size_t rebar_size; 55 - struct resource *root_res; 56 - int max_size, i; 57 - u32 pci_cmd; 58 - 59 - /* gather some relevant info */ 60 - current_size = pci_resource_len(pdev, LMEM_BAR); 61 - 62 - if (force_vram_bar_size < 0) 63 - return; 64 - 65 - /* set to a specific size? */ 66 - if (force_vram_bar_size) { 67 - rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * 68 - (resource_size_t)SZ_1M); 69 - 70 - if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { 71 - drm_info(&xe->drm, 72 - "Requested size: %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", 73 - (u64)pci_rebar_size_to_bytes(rebar_size) >> 20, 74 - pci_rebar_get_possible_sizes(pdev, LMEM_BAR), 75 - (u64)current_size >> 20); 76 - return; 77 - } 78 - 79 - rebar_size = pci_rebar_size_to_bytes(rebar_size); 80 - if (rebar_size == current_size) 81 - return; 82 - } else { 83 - max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); 84 - if (max_size < 0) 85 - return; 86 - rebar_size = pci_rebar_size_to_bytes(max_size); 87 - 88 - /* only resize if larger than current */ 89 - if (rebar_size <= current_size) 90 - return; 91 - } 92 - 93 - drm_info(&xe->drm, "Attempting to resize bar from %lluMiB -> %lluMiB\n", 94 - (u64)current_size >> 20, (u64)rebar_size >> 20); 95 - 96 - while (root->parent) 97 - root = root->parent; 98 - 99 - pci_bus_for_each_resource(root, root_res, i) { 100 - if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 101 - (u64)root_res->start > 0x100000000ul) 102 - break; 103 - } 104 - 105 - if (!root_res) { 106 - drm_info(&xe->drm, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); 107 - return; 108 - } 109 - 110 - pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); 111 - pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); 112 - 113 - resize_bar(xe, LMEM_BAR, rebar_size); 114 - 115 - pci_assign_unassigned_bus_resources(pdev->bus); 116 - pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); 117 - } 118 - 119 28 static bool resource_is_valid(struct pci_dev *pdev, int bar) 120 29 { 121 30 if (!pci_resource_flags(pdev, bar))

-1

drivers/gpu/drm/xe/xe_vram.h

··· 11 11 struct xe_device; 12 12 struct xe_vram_region; 13 13 14 - void xe_vram_resize_bar(struct xe_device *xe); 15 14 int xe_vram_probe(struct xe_device *xe); 16 15 17 16 struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement);

+2 -13

drivers/gpu/drm/xe/xe_vram_types.h

··· 66 66 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 67 67 /** @migrate: Back pointer to migrate */ 68 68 struct xe_migrate *migrate; 69 - /** @pagemap: Used to remap device memory as ZONE_DEVICE */ 70 - struct dev_pagemap pagemap; 71 - /** 72 - * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory 73 - * pages of this tile. 74 - */ 75 - struct drm_pagemap dpagemap; 76 - /** 77 - * @hpa_base: base host physical address 78 - * 79 - * This is generated when remap device memory as ZONE_DEVICE 80 - */ 81 - resource_size_t hpa_base; 69 + /** @dpagemap_cache: drm_pagemap cache. */ 70 + struct drm_pagemap_cache *dpagemap_cache; 82 71 #endif 83 72 }; 84 73

+4 -2

drivers/gpu/drm/xe/xe_vsec.c

··· 158 158 159 159 guard(mutex)(&xe->pmt.lock); 160 160 161 + if (!xe->soc_remapper.set_telem_region) 162 + return -ENODEV; 163 + 161 164 /* indicate that we are not at an appropriate power level */ 162 165 if (!xe_pm_runtime_get_if_active(xe)) 163 166 return -ENODATA; 164 167 165 168 /* set SoC re-mapper index register based on GUID memory region */ 166 - xe_mmio_rmw32(xe_root_tile_mmio(xe), SG_REMAP_INDEX1, SG_REMAP_BITS, 167 - REG_FIELD_PREP(SG_REMAP_BITS, mem_region)); 169 + xe->soc_remapper.set_telem_region(xe, mem_region); 168 170 169 171 memcpy_fromio(data, telem_addr, count); 170 172 xe_pm_runtime_put(xe);

+29

include/drm/drm_gpusvm.h

··· 328 328 struct drm_gpusvm_pages *svm_pages, 329 329 unsigned long npages); 330 330 331 + /** 332 + * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function. 333 + * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible. 334 + * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as 335 + * the @pagemap argument to the drm_gpusvm_scan_mm() function. 336 + * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is 337 + * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to 338 + * the same device private owner. 339 + * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages. 340 + * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least 341 + * two different struct dev_pagemaps. All pages belong to the same device private 342 + * owner. 343 + * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages 344 + * and device-private pages. All device-private pages belong to the same device 345 + * private owner. 346 + */ 347 + enum drm_gpusvm_scan_result { 348 + DRM_GPUSVM_SCAN_UNPOPULATED, 349 + DRM_GPUSVM_SCAN_EQUAL, 350 + DRM_GPUSVM_SCAN_OTHER, 351 + DRM_GPUSVM_SCAN_SYSTEM, 352 + DRM_GPUSVM_SCAN_MIXED_DEVICE, 353 + DRM_GPUSVM_SCAN_MIXED, 354 + }; 355 + 356 + enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, 357 + void *dev_private_owner, 358 + const struct dev_pagemap *pagemap); 359 + 331 360 #ifdef CONFIG_LOCKDEP 332 361 /** 333 362 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM

+121 -7

include/drm/drm_pagemap.h

··· 8 8 9 9 #define NR_PAGES(order) (1U << (order)) 10 10 11 + struct dma_fence; 11 12 struct drm_pagemap; 13 + struct drm_pagemap_cache; 14 + struct drm_pagemap_dev_hold; 12 15 struct drm_pagemap_zdd; 13 16 struct device; 14 17 ··· 126 123 unsigned long start, unsigned long end, 127 124 struct mm_struct *mm, 128 125 unsigned long timeslice_ms); 126 + /** 127 + * @destroy: Destroy the drm_pagemap and associated resources. 128 + * @dpagemap: The drm_pagemap to destroy. 129 + * @is_atomic_or_reclaim: The function may be called from 130 + * atomic- or reclaim context. 131 + * 132 + * The implementation should take care not to attempt to 133 + * destroy resources that may already have been destroyed 134 + * using devm_ callbacks, since this function may be called 135 + * after the underlying struct device has been unbound. 136 + * If the implementation defers the execution to a work item 137 + * to avoid locking issues, then it must make sure the work 138 + * items are flushed before module exit. If the destroy call 139 + * happens after the provider's pci_remove() callback has 140 + * been executed, a module reference and drm device reference is 141 + * held across the destroy callback. 142 + */ 143 + void (*destroy)(struct drm_pagemap *dpagemap, 144 + bool is_atomic_or_reclaim); 129 145 }; 130 146 131 147 /** 132 148 * struct drm_pagemap: Additional information for a struct dev_pagemap 133 149 * used for device p2p handshaking. 134 150 * @ops: The struct drm_pagemap_ops. 135 - * @dev: The struct drevice owning the device-private memory. 151 + * @ref: Reference count. 152 + * @drm: The struct drm device owning the device-private memory. 153 + * @pagemap: Pointer to the underlying dev_pagemap. 154 + * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for 155 + * device referencing. 156 + * @cache: Back-pointer to the &struct drm_pagemap_cache used for this 157 + * &struct drm_pagemap. May be NULL if no cache is used. 158 + * @shrink_link: Link into the shrinker's list of drm_pagemaps. Only 159 + * used if also using a pagemap cache. 136 160 */ 137 161 struct drm_pagemap { 138 162 const struct drm_pagemap_ops *ops; 139 - struct device *dev; 163 + struct kref ref; 164 + struct drm_device *drm; 165 + struct dev_pagemap *pagemap; 166 + struct drm_pagemap_dev_hold *dev_hold; 167 + struct drm_pagemap_cache *cache; 168 + struct list_head shrink_link; 140 169 }; 141 170 142 171 struct drm_pagemap_devmem; ··· 209 174 * @pages: Pointer to array of device memory pages (destination) 210 175 * @pagemap_addr: Pointer to array of DMA information (source) 211 176 * @npages: Number of pages to copy 177 + * @pre_migrate_fence: dma-fence to wait for before migration start. 178 + * May be NULL. 212 179 * 213 180 * Copy pages to device memory. If the order of a @pagemap_addr entry 214 181 * is greater than 0, the entry is populated but subsequent entries ··· 220 183 */ 221 184 int (*copy_to_devmem)(struct page **pages, 222 185 struct drm_pagemap_addr *pagemap_addr, 223 - unsigned long npages); 186 + unsigned long npages, 187 + struct dma_fence *pre_migrate_fence); 224 188 225 189 /** 226 190 * @copy_to_ram: Copy to system RAM (required for migration) 227 191 * @pages: Pointer to array of device memory pages (source) 228 192 * @pagemap_addr: Pointer to array of DMA information (destination) 229 193 * @npages: Number of pages to copy 194 + * @pre_migrate_fence: dma-fence to wait for before migration start. 195 + * May be NULL. 230 196 * 231 197 * Copy pages to system RAM. If the order of a @pagemap_addr entry 232 198 * is greater than 0, the entry is populated but subsequent entries ··· 239 199 */ 240 200 int (*copy_to_ram)(struct page **pages, 241 201 struct drm_pagemap_addr *pagemap_addr, 242 - unsigned long npages); 202 + unsigned long npages, 203 + struct dma_fence *pre_migrate_fence); 243 204 }; 205 + 206 + int drm_pagemap_init(struct drm_pagemap *dpagemap, 207 + struct dev_pagemap *pagemap, 208 + struct drm_device *drm, 209 + const struct drm_pagemap_ops *ops); 210 + 211 + struct drm_pagemap *drm_pagemap_create(struct drm_device *drm, 212 + struct dev_pagemap *pagemap, 213 + const struct drm_pagemap_ops *ops); 214 + 215 + #if IS_ENABLED(CONFIG_DRM_GPUSVM) 216 + 217 + void drm_pagemap_put(struct drm_pagemap *dpagemap); 218 + 219 + #else 220 + 221 + static inline void drm_pagemap_put(struct drm_pagemap *dpagemap) 222 + { 223 + } 224 + 225 + #endif /* IS_ENABLED(CONFIG_DRM_GPUSVM) */ 226 + 227 + /** 228 + * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap 229 + * @dpagemap: Pointer to the struct drm_pagemap, or NULL. 230 + * 231 + * Return: Pointer to the struct drm_pagemap, or NULL. 232 + */ 233 + static inline struct drm_pagemap * 234 + drm_pagemap_get(struct drm_pagemap *dpagemap) 235 + { 236 + if (likely(dpagemap)) 237 + kref_get(&dpagemap->ref); 238 + 239 + return dpagemap; 240 + } 241 + 242 + /** 243 + * drm_pagemap_get_unless_zero() - Obtain a reference on a struct drm_pagemap 244 + * unless the current reference count is zero. 245 + * @dpagemap: Pointer to the drm_pagemap or NULL. 246 + * 247 + * Return: A pointer to @dpagemap if the reference count was successfully 248 + * incremented. NULL if @dpagemap was NULL, or its refcount was 0. 249 + */ 250 + static inline struct drm_pagemap * __must_check 251 + drm_pagemap_get_unless_zero(struct drm_pagemap *dpagemap) 252 + { 253 + return (dpagemap && kref_get_unless_zero(&dpagemap->ref)) ? dpagemap : NULL; 254 + } 244 255 245 256 /** 246 257 * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation ··· 303 212 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. 304 213 * @size: Size of device memory allocation 305 214 * @timeslice_expiration: Timeslice expiration in jiffies 215 + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 216 + * (May be NULL). 306 217 */ 307 218 struct drm_pagemap_devmem { 308 219 struct device *dev; ··· 314 221 struct drm_pagemap *dpagemap; 315 222 size_t size; 316 223 u64 timeslice_expiration; 224 + struct dma_fence *pre_migrate_fence; 225 + }; 226 + 227 + /** 228 + * struct drm_pagemap_migrate_details - Details to govern migration. 229 + * @timeslice_ms: The time requested for the migrated pagemap pages to 230 + * be present in @mm before being allowed to be migrated back. 231 + * @can_migrate_same_pagemap: Whether the copy function as indicated by 232 + * the @source_peer_migrates flag, can migrate device pages within a 233 + * single drm_pagemap. 234 + * @source_peer_migrates: Whether on p2p migration, The source drm_pagemap 235 + * should use the copy_to_ram() callback rather than the destination 236 + * drm_pagemap should use the copy_to_devmem() callback. 237 + */ 238 + struct drm_pagemap_migrate_details { 239 + unsigned long timeslice_ms; 240 + u32 can_migrate_same_pagemap : 1; 241 + u32 source_peer_migrates : 1; 317 242 }; 318 243 319 244 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, 320 245 struct mm_struct *mm, 321 246 unsigned long start, unsigned long end, 322 - unsigned long timeslice_ms, 323 - void *pgmap_owner); 247 + const struct drm_pagemap_migrate_details *mdetails); 324 248 325 249 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation); 326 250 ··· 348 238 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 349 239 struct device *dev, struct mm_struct *mm, 350 240 const struct drm_pagemap_devmem_ops *ops, 351 - struct drm_pagemap *dpagemap, size_t size); 241 + struct drm_pagemap *dpagemap, size_t size, 242 + struct dma_fence *pre_migrate_fence); 352 243 353 244 int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, 354 245 unsigned long start, unsigned long end, 355 246 struct mm_struct *mm, 356 247 unsigned long timeslice_ms); 357 248 249 + void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim); 250 + 251 + int drm_pagemap_reinit(struct drm_pagemap *dpagemap); 358 252 #endif

+92

include/drm/drm_pagemap_util.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef _DRM_PAGEMAP_UTIL_H_ 7 + #define _DRM_PAGEMAP_UTIL_H_ 8 + 9 + #include <linux/list.h> 10 + #include <linux/mutex.h> 11 + 12 + struct drm_device; 13 + struct drm_pagemap; 14 + struct drm_pagemap_cache; 15 + struct drm_pagemap_owner; 16 + struct drm_pagemap_shrinker; 17 + 18 + /** 19 + * struct drm_pagemap_peer - Structure representing a fast interconnect peer 20 + * @list: Pointer to a &struct drm_pagemap_owner_list used to keep track of peers 21 + * @link: List link for @list's list of peers. 22 + * @owner: Pointer to a &struct drm_pagemap_owner, common for a set of peers having 23 + * fast interconnects. 24 + * @private: Pointer private to the struct embedding this struct. 25 + */ 26 + struct drm_pagemap_peer { 27 + struct drm_pagemap_owner_list *list; 28 + struct list_head link; 29 + struct drm_pagemap_owner *owner; 30 + void *private; 31 + }; 32 + 33 + /** 34 + * struct drm_pagemap_owner_list - Keeping track of peers and owners 35 + * @peer: List of peers. 36 + * 37 + * The owner list defines the scope where we identify peers having fast interconnects 38 + * and a common owner. Typically a driver has a single global owner list to 39 + * keep track of common owners for the driver's pagemaps. 40 + */ 41 + struct drm_pagemap_owner_list { 42 + /** @lock: Mutex protecting the @peers list. */ 43 + struct mutex lock; 44 + /** @peers: List of peers. */ 45 + struct list_head peers; 46 + }; 47 + 48 + /* 49 + * Convenience macro to define an owner list. 50 + * Typically the owner list statically declared 51 + * driver-wide. 52 + */ 53 + #define DRM_PAGEMAP_OWNER_LIST_DEFINE(_name) \ 54 + struct drm_pagemap_owner_list _name = { \ 55 + .lock = __MUTEX_INITIALIZER((_name).lock), \ 56 + .peers = LIST_HEAD_INIT((_name).peers) } 57 + 58 + void drm_pagemap_shrinker_add(struct drm_pagemap *dpagemap); 59 + 60 + int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache *cache); 61 + 62 + void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache *cache); 63 + 64 + struct drm_pagemap_shrinker *drm_pagemap_shrinker_create_devm(struct drm_device *drm); 65 + 66 + struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct drm_pagemap_shrinker *shrinker); 67 + 68 + struct drm_pagemap *drm_pagemap_get_from_cache(struct drm_pagemap_cache *cache); 69 + 70 + void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache *cache, struct drm_pagemap *dpagemap); 71 + 72 + struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct drm_pagemap_cache *cache); 73 + 74 + #ifdef CONFIG_PROVE_LOCKING 75 + 76 + void drm_pagemap_shrinker_might_lock(struct drm_pagemap *dpagemap); 77 + 78 + #else 79 + 80 + static inline void drm_pagemap_shrinker_might_lock(struct drm_pagemap *dpagemap) 81 + { 82 + } 83 + 84 + #endif /* CONFIG_PROVE_LOCKING */ 85 + 86 + void drm_pagemap_release_owner(struct drm_pagemap_peer *peer); 87 + 88 + int drm_pagemap_acquire_owner(struct drm_pagemap_peer *peer, 89 + struct drm_pagemap_owner_list *owner_list, 90 + bool (*has_interconnect)(struct drm_pagemap_peer *peer1, 91 + struct drm_pagemap_peer *peer2)); 92 + #endif

+15 -3

include/uapi/drm/xe_drm.h

··· 2123 2123 struct { 2124 2124 #define DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE 0 2125 2125 #define DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM -1 2126 - /** @preferred_mem_loc.devmem_fd: fd for preferred loc */ 2126 + /** 2127 + * @preferred_mem_loc.devmem_fd: 2128 + * Device file-descriptor of the device where the 2129 + * preferred memory is located, or one of the 2130 + * above special values. Please also see 2131 + * @preferred_mem_loc.region_instance below. 2132 + */ 2127 2133 __u32 devmem_fd; 2128 2134 2129 2135 #define DRM_XE_MIGRATE_ALL_PAGES 0 ··· 2137 2131 /** @preferred_mem_loc.migration_policy: Page migration policy */ 2138 2132 __u16 migration_policy; 2139 2133 2140 - /** @preferred_mem_loc.pad : MBZ */ 2141 - __u16 pad; 2134 + /** 2135 + * @preferred_mem_loc.region_instance : Region instance. 2136 + * MBZ if @devmem_fd <= &DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE. 2137 + * Otherwise should point to the desired device 2138 + * VRAM instance of the device indicated by 2139 + * @preferred_mem_loc.devmem_fd. 2140 + */ 2141 + __u16 region_instance; 2142 2142 2143 2143 /** @preferred_mem_loc.reserved : Reserved */ 2144 2144 __u64 reserved;

Configure Feed

Configure Feed