Merge tag 'drm-xe-fixes-2025-12-30' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-fixes

+13 -4

drivers/gpu/drm/drm_pagemap.c

··· 3 3 * Copyright © 2024-2025 Intel Corporation 4 4 */ 5 5 6 + #include <linux/dma-fence.h> 6 7 #include <linux/dma-mapping.h> 7 8 #include <linux/migrate.h> 8 9 #include <linux/pagemap.h> ··· 409 408 drm_pagemap_get_devmem_page(page, zdd); 410 409 } 411 410 412 - err = ops->copy_to_devmem(pages, pagemap_addr, npages); 411 + err = ops->copy_to_devmem(pages, pagemap_addr, npages, 412 + devmem_allocation->pre_migrate_fence); 413 413 if (err) 414 414 goto err_finalize; 415 + 416 + dma_fence_put(devmem_allocation->pre_migrate_fence); 417 + devmem_allocation->pre_migrate_fence = NULL; 415 418 416 419 /* Upon success bind devmem allocation to range and zdd */ 417 420 devmem_allocation->timeslice_expiration = get_jiffies_64() + ··· 601 596 for (i = 0; i < npages; ++i) 602 597 pages[i] = migrate_pfn_to_page(src[i]); 603 598 604 - err = ops->copy_to_ram(pages, pagemap_addr, npages); 599 + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 605 600 if (err) 606 601 goto err_finalize; 607 602 ··· 737 732 for (i = 0; i < npages; ++i) 738 733 pages[i] = migrate_pfn_to_page(migrate.src[i]); 739 734 740 - err = ops->copy_to_ram(pages, pagemap_addr, npages); 735 + err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 741 736 if (err) 742 737 goto err_finalize; 743 738 ··· 818 813 * @ops: Pointer to the operations structure for GPU SVM device memory 819 814 * @dpagemap: The struct drm_pagemap we're allocating from. 820 815 * @size: Size of device memory allocation 816 + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 817 + * (May be NULL). 821 818 */ 822 819 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 823 820 struct device *dev, struct mm_struct *mm, 824 821 const struct drm_pagemap_devmem_ops *ops, 825 - struct drm_pagemap *dpagemap, size_t size) 822 + struct drm_pagemap *dpagemap, size_t size, 823 + struct dma_fence *pre_migrate_fence) 826 824 { 827 825 init_completion(&devmem_allocation->detached); 828 826 devmem_allocation->dev = dev; ··· 833 825 devmem_allocation->ops = ops; 834 826 devmem_allocation->dpagemap = dpagemap; 835 827 devmem_allocation->size = size; 828 + devmem_allocation->pre_migrate_fence = pre_migrate_fence; 836 829 } 837 830 EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init); 838 831

+10 -4

drivers/gpu/drm/xe/xe_guc_ct.c

··· 104 104 { 105 105 g2h_fence->cancel = true; 106 106 g2h_fence->fail = true; 107 - g2h_fence->done = true; 107 + 108 + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 109 + WRITE_ONCE(g2h_fence->done, true); 108 110 } 109 111 110 112 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) ··· 1205 1203 return ret; 1206 1204 } 1207 1205 1208 - ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1206 + /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response 1207 + * and g2h_fence_cancel. 1208 + */ 1209 + ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); 1209 1210 if (!ret) { 1210 1211 LNL_FLUSH_WORK(&ct->g2h_worker); 1211 - if (g2h_fence.done) { 1212 + if (READ_ONCE(g2h_fence.done)) { 1212 1213 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1213 1214 g2h_fence.seqno, action[0]); 1214 1215 ret = 1; ··· 1459 1454 1460 1455 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1461 1456 1462 - g2h_fence->done = true; 1457 + /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 1458 + WRITE_ONCE(g2h_fence->done, true); 1463 1459 smp_mb(); 1464 1460 1465 1461 wake_up_all(&ct->g2h_fence_wq);

+20 -5

drivers/gpu/drm/xe/xe_migrate.c

··· 2062 2062 unsigned long sram_offset, 2063 2063 struct drm_pagemap_addr *sram_addr, 2064 2064 u64 vram_addr, 2065 + struct dma_fence *deps, 2065 2066 const enum xe_migrate_copy_dir dir) 2066 2067 { 2067 2068 struct xe_gt *gt = m->tile->primary_gt; ··· 2151 2150 2152 2151 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); 2153 2152 2153 + if (deps && !dma_fence_is_signaled(deps)) { 2154 + dma_fence_get(deps); 2155 + err = drm_sched_job_add_dependency(&job->drm, deps); 2156 + if (err) 2157 + dma_fence_wait(deps, false); 2158 + err = 0; 2159 + } 2160 + 2154 2161 mutex_lock(&m->job_mutex); 2155 2162 xe_sched_job_arm(job); 2156 2163 fence = dma_fence_get(&job->drm.s_fence->finished); ··· 2184 2175 * @npages: Number of pages to migrate. 2185 2176 * @src_addr: Array of DMA information (source of migrate) 2186 2177 * @dst_addr: Device physical address of VRAM (destination of migrate) 2178 + * @deps: struct dma_fence representing the dependencies that need 2179 + * to be signaled before migration. 2187 2180 * 2188 2181 * Copy from an array dma addresses to a VRAM device physical address 2189 2182 * ··· 2195 2184 struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, 2196 2185 unsigned long npages, 2197 2186 struct drm_pagemap_addr *src_addr, 2198 - u64 dst_addr) 2187 + u64 dst_addr, 2188 + struct dma_fence *deps) 2199 2189 { 2200 2190 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, 2201 - XE_MIGRATE_COPY_TO_VRAM); 2191 + deps, XE_MIGRATE_COPY_TO_VRAM); 2202 2192 } 2203 2193 2204 2194 /** ··· 2208 2196 * @npages: Number of pages to migrate. 2209 2197 * @src_addr: Device physical address of VRAM (source of migrate) 2210 2198 * @dst_addr: Array of DMA information (destination of migrate) 2199 + * @deps: struct dma_fence representing the dependencies that need 2200 + * to be signaled before migration. 2211 2201 * 2212 2202 * Copy from a VRAM device physical address to an array dma addresses 2213 2203 * ··· 2219 2205 struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, 2220 2206 unsigned long npages, 2221 2207 u64 src_addr, 2222 - struct drm_pagemap_addr *dst_addr) 2208 + struct drm_pagemap_addr *dst_addr, 2209 + struct dma_fence *deps) 2223 2210 { 2224 2211 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, 2225 - XE_MIGRATE_COPY_TO_SRAM); 2212 + deps, XE_MIGRATE_COPY_TO_SRAM); 2226 2213 } 2227 2214 2228 2215 static void xe_migrate_dma_unmap(struct xe_device *xe, ··· 2399 2384 __fence = xe_migrate_vram(m, current_bytes, 2400 2385 (unsigned long)buf & ~PAGE_MASK, 2401 2386 &pagemap_addr[current_page], 2402 - vram_addr, write ? 2387 + vram_addr, NULL, write ? 2403 2388 XE_MIGRATE_COPY_TO_VRAM : 2404 2389 XE_MIGRATE_COPY_TO_SRAM); 2405 2390 if (IS_ERR(__fence)) {

+4 -2

drivers/gpu/drm/xe/xe_migrate.h

··· 116 116 struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, 117 117 unsigned long npages, 118 118 struct drm_pagemap_addr *src_addr, 119 - u64 dst_addr); 119 + u64 dst_addr, 120 + struct dma_fence *deps); 120 121 121 122 struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, 122 123 unsigned long npages, 123 124 u64 src_addr, 124 - struct drm_pagemap_addr *dst_addr); 125 + struct drm_pagemap_addr *dst_addr, 126 + struct dma_fence *deps); 125 127 126 128 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 127 129 struct xe_bo *src_bo,

+38 -13

drivers/gpu/drm/xe/xe_svm.c

··· 476 476 477 477 static int xe_svm_copy(struct page **pages, 478 478 struct drm_pagemap_addr *pagemap_addr, 479 - unsigned long npages, const enum xe_svm_copy_dir dir) 479 + unsigned long npages, const enum xe_svm_copy_dir dir, 480 + struct dma_fence *pre_migrate_fence) 480 481 { 481 482 struct xe_vram_region *vr = NULL; 482 483 struct xe_gt *gt = NULL; ··· 566 565 __fence = xe_migrate_from_vram(vr->migrate, 567 566 i - pos + incr, 568 567 vram_addr, 569 - &pagemap_addr[pos]); 568 + &pagemap_addr[pos], 569 + pre_migrate_fence); 570 570 } else { 571 571 vm_dbg(&xe->drm, 572 572 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", ··· 576 574 __fence = xe_migrate_to_vram(vr->migrate, 577 575 i - pos + incr, 578 576 &pagemap_addr[pos], 579 - vram_addr); 577 + vram_addr, 578 + pre_migrate_fence); 580 579 } 581 580 if (IS_ERR(__fence)) { 582 581 err = PTR_ERR(__fence); 583 582 goto err_out; 584 583 } 585 - 584 + pre_migrate_fence = NULL; 586 585 dma_fence_put(fence); 587 586 fence = __fence; 588 587 } ··· 606 603 vram_addr, (u64)pagemap_addr[pos].addr, 1); 607 604 __fence = xe_migrate_from_vram(vr->migrate, 1, 608 605 vram_addr, 609 - &pagemap_addr[pos]); 606 + &pagemap_addr[pos], 607 + pre_migrate_fence); 610 608 } else { 611 609 vm_dbg(&xe->drm, 612 610 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", 613 611 (u64)pagemap_addr[pos].addr, vram_addr, 1); 614 612 __fence = xe_migrate_to_vram(vr->migrate, 1, 615 613 &pagemap_addr[pos], 616 - vram_addr); 614 + vram_addr, 615 + pre_migrate_fence); 617 616 } 618 617 if (IS_ERR(__fence)) { 619 618 err = PTR_ERR(__fence); 620 619 goto err_out; 621 620 } 622 - 621 + pre_migrate_fence = NULL; 623 622 dma_fence_put(fence); 624 623 fence = __fence; 625 624 } ··· 634 629 dma_fence_wait(fence, false); 635 630 dma_fence_put(fence); 636 631 } 632 + if (pre_migrate_fence) 633 + dma_fence_wait(pre_migrate_fence, false); 637 634 638 635 /* 639 636 * XXX: We can't derive the GT here (or anywhere in this functions, but ··· 652 645 653 646 static int xe_svm_copy_to_devmem(struct page **pages, 654 647 struct drm_pagemap_addr *pagemap_addr, 655 - unsigned long npages) 648 + unsigned long npages, 649 + struct dma_fence *pre_migrate_fence) 656 650 { 657 - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM); 651 + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM, 652 + pre_migrate_fence); 658 653 } 659 654 660 655 static int xe_svm_copy_to_ram(struct page **pages, 661 656 struct drm_pagemap_addr *pagemap_addr, 662 - unsigned long npages) 657 + unsigned long npages, 658 + struct dma_fence *pre_migrate_fence) 663 659 { 664 - return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM); 660 + return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM, 661 + pre_migrate_fence); 665 662 } 666 663 667 664 static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) ··· 678 667 struct xe_bo *bo = to_xe_bo(devmem_allocation); 679 668 struct xe_device *xe = xe_bo_device(bo); 680 669 670 + dma_fence_put(devmem_allocation->pre_migrate_fence); 681 671 xe_bo_put_async(bo); 682 672 xe_pm_runtime_put(xe); 683 673 } ··· 873 861 unsigned long timeslice_ms) 874 862 { 875 863 struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); 864 + struct dma_fence *pre_migrate_fence = NULL; 876 865 struct xe_device *xe = vr->xe; 877 866 struct device *dev = xe->drm.dev; 878 867 struct drm_buddy_block *block; ··· 900 887 break; 901 888 } 902 889 890 + /* Ensure that any clearing or async eviction will complete before migration. */ 891 + if (!dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL)) { 892 + err = dma_resv_get_singleton(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 893 + &pre_migrate_fence); 894 + if (err) 895 + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, 896 + false, MAX_SCHEDULE_TIMEOUT); 897 + else if (pre_migrate_fence) 898 + dma_fence_enable_sw_signaling(pre_migrate_fence); 899 + } 900 + 903 901 drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, 904 - &dpagemap_devmem_ops, dpagemap, end - start); 902 + &dpagemap_devmem_ops, dpagemap, end - start, 903 + pre_migrate_fence); 905 904 906 905 blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; 907 906 list_for_each_entry(block, blocks, link) ··· 966 941 xe_assert(vm->xe, IS_DGFX(vm->xe)); 967 942 968 943 if (xe_svm_range_in_vram(range)) { 969 - drm_info(&vm->xe->drm, "Range is already in VRAM\n"); 944 + drm_dbg(&vm->xe->drm, "Range is already in VRAM\n"); 970 945 return false; 971 946 } 972 947

+14 -3

include/drm/drm_pagemap.h

··· 8 8 9 9 #define NR_PAGES(order) (1U << (order)) 10 10 11 + struct dma_fence; 11 12 struct drm_pagemap; 12 13 struct drm_pagemap_zdd; 13 14 struct device; ··· 175 174 * @pages: Pointer to array of device memory pages (destination) 176 175 * @pagemap_addr: Pointer to array of DMA information (source) 177 176 * @npages: Number of pages to copy 177 + * @pre_migrate_fence: dma-fence to wait for before migration start. 178 + * May be NULL. 178 179 * 179 180 * Copy pages to device memory. If the order of a @pagemap_addr entry 180 181 * is greater than 0, the entry is populated but subsequent entries ··· 186 183 */ 187 184 int (*copy_to_devmem)(struct page **pages, 188 185 struct drm_pagemap_addr *pagemap_addr, 189 - unsigned long npages); 186 + unsigned long npages, 187 + struct dma_fence *pre_migrate_fence); 190 188 191 189 /** 192 190 * @copy_to_ram: Copy to system RAM (required for migration) 193 191 * @pages: Pointer to array of device memory pages (source) 194 192 * @pagemap_addr: Pointer to array of DMA information (destination) 195 193 * @npages: Number of pages to copy 194 + * @pre_migrate_fence: dma-fence to wait for before migration start. 195 + * May be NULL. 196 196 * 197 197 * Copy pages to system RAM. If the order of a @pagemap_addr entry 198 198 * is greater than 0, the entry is populated but subsequent entries ··· 205 199 */ 206 200 int (*copy_to_ram)(struct page **pages, 207 201 struct drm_pagemap_addr *pagemap_addr, 208 - unsigned long npages); 202 + unsigned long npages, 203 + struct dma_fence *pre_migrate_fence); 209 204 }; 210 205 211 206 /** ··· 219 212 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. 220 213 * @size: Size of device memory allocation 221 214 * @timeslice_expiration: Timeslice expiration in jiffies 215 + * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 216 + * (May be NULL). 222 217 */ 223 218 struct drm_pagemap_devmem { 224 219 struct device *dev; ··· 230 221 struct drm_pagemap *dpagemap; 231 222 size_t size; 232 223 u64 timeslice_expiration; 224 + struct dma_fence *pre_migrate_fence; 233 225 }; 234 226 235 227 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, ··· 248 238 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 249 239 struct device *dev, struct mm_struct *mm, 250 240 const struct drm_pagemap_devmem_ops *ops, 251 - struct drm_pagemap *dpagemap, size_t size); 241 + struct drm_pagemap *dpagemap, size_t size, 242 + struct dma_fence *pre_migrate_fence); 252 243 253 244 int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, 254 245 unsigned long start, unsigned long end,

Configure Feed

Configure Feed