Merge tag 'drm-intel-gt-next-2022-03-03' of git://anongit.freedesktop.org/drm/drm-intel into drm-next

+1

drivers/gpu/drm/i915/Kconfig

··· 3 3 tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" 4 4 depends on DRM 5 5 depends on X86 && PCI 6 + depends on !PREEMPT_RT 6 7 select INTEL_GTT 7 8 select INTERVAL_TREE 8 9 # we need shmfs for the swappable backing store, and in particular

-8

drivers/gpu/drm/i915/Kconfig.unstable

··· 19 19 Recommended for driver developers _only_. 20 20 21 21 If in the slightest bit of doubt, say "N". 22 - 23 - config DRM_I915_UNSTABLE_FAKE_LMEM 24 - bool "Enable the experimental fake lmem" 25 - depends on DRM_I915_UNSTABLE 26 - default n 27 - help 28 - Convert some system memory into a fake local memory region for 29 - testing.

+1 -1

drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c

··· 1321 1321 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 1322 1322 1323 1323 if (crtc_state->port_clock > 270000) { 1324 - if (IS_TGL_U(dev_priv) || IS_TGL_Y(dev_priv)) { 1324 + if (IS_TGL_UY(dev_priv)) { 1325 1325 return intel_get_buf_trans(&tgl_uy_combo_phy_trans_dp_hbr2, 1326 1326 n_entries); 1327 1327 } else {

+1 -1

drivers/gpu/drm/i915/display/intel_fbdev.c

··· 265 265 struct intel_memory_region *mem = obj->mm.region; 266 266 267 267 info->apertures->ranges[0].base = mem->io_start; 268 - info->apertures->ranges[0].size = mem->total; 268 + info->apertures->ranges[0].size = mem->io_size; 269 269 270 270 /* Use fbdev's framebuffer from lmem for discrete */ 271 271 info->fix.smem_start =

+10

drivers/gpu/drm/i915/gem/i915_gem_context.c

··· 670 670 goto out_err; 671 671 } 672 672 673 + /* 674 + * We don't support breadcrumb handshake on these 675 + * classes 676 + */ 677 + if (siblings[n]->class == RENDER_CLASS || 678 + siblings[n]->class == COMPUTE_CLASS) { 679 + err = -EINVAL; 680 + goto out_err; 681 + } 682 + 673 683 if (n) { 674 684 if (prev_engine.engine_class != 675 685 ci.engine_class) {

+1 -13

drivers/gpu/drm/i915/gem/i915_gem_object.c

··· 272 272 if (!list_empty(&obj->vma.list)) { 273 273 struct i915_vma *vma; 274 274 275 - /* 276 - * Note that the vma keeps an object reference while 277 - * it is active, so it *should* not sleep while we 278 - * destroy it. Our debug code errs insits it *might*. 279 - * For the moment, play along. 280 - */ 281 275 spin_lock(&obj->vma.lock); 282 276 while ((vma = list_first_entry_or_null(&obj->vma.list, 283 277 struct i915_vma, ··· 279 285 GEM_BUG_ON(vma->obj != obj); 280 286 spin_unlock(&obj->vma.lock); 281 287 282 - /* Verify that the vma is unbound under the vm mutex. */ 283 - mutex_lock(&vma->vm->mutex); 284 - atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 285 - __i915_vma_unbind(vma); 286 - mutex_unlock(&vma->vm->mutex); 287 - 288 - __i915_vma_put(vma); 288 + i915_vma_destroy(vma); 289 289 290 290 spin_lock(&obj->vma.lock); 291 291 }

+12 -5

drivers/gpu/drm/i915/gem/i915_gem_object_types.h

··· 319 319 #define I915_BO_ALLOC_PM_VOLATILE BIT(4) 320 320 /* Object needs to be restored early using memcpy during resume */ 321 321 #define I915_BO_ALLOC_PM_EARLY BIT(5) 322 + /* 323 + * Object is likely never accessed by the CPU. This will prioritise the BO to be 324 + * allocated in the non-mappable portion of lmem. This is merely a hint, and if 325 + * dealing with userspace objects the CPU fault handler is free to ignore this. 326 + */ 327 + #define I915_BO_ALLOC_GPU_ONLY BIT(6) 322 328 #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ 323 329 I915_BO_ALLOC_VOLATILE | \ 324 330 I915_BO_ALLOC_CPU_CLEAR | \ 325 331 I915_BO_ALLOC_USER | \ 326 332 I915_BO_ALLOC_PM_VOLATILE | \ 327 - I915_BO_ALLOC_PM_EARLY) 328 - #define I915_BO_READONLY BIT(6) 329 - #define I915_TILING_QUIRK_BIT 7 /* unknown swizzling; do not release! */ 330 - #define I915_BO_PROTECTED BIT(8) 331 - #define I915_BO_WAS_BOUND_BIT 9 333 + I915_BO_ALLOC_PM_EARLY | \ 334 + I915_BO_ALLOC_GPU_ONLY) 335 + #define I915_BO_READONLY BIT(7) 336 + #define I915_TILING_QUIRK_BIT 8 /* unknown swizzling; do not release! */ 337 + #define I915_BO_PROTECTED BIT(9) 338 + #define I915_BO_WAS_BOUND_BIT 10 332 339 /** 333 340 * @mem_flags - Mutable placement-related flags 334 341 *

+3

drivers/gpu/drm/i915/gem/i915_gem_pages.c

··· 358 358 !i915_gem_object_has_iomem(obj)) 359 359 return ERR_PTR(-ENXIO); 360 360 361 + if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY)) 362 + return ERR_PTR(-EINVAL); 363 + 361 364 assert_object_held(obj); 362 365 363 366 pinned = !(type & I915_MAP_OVERRIDE);

+16

drivers/gpu/drm/i915/gem/i915_gem_region.c

··· 45 45 46 46 GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS); 47 47 48 + if (WARN_ON_ONCE(flags & I915_BO_ALLOC_GPU_ONLY && 49 + (flags & I915_BO_ALLOC_CPU_CLEAR || 50 + flags & I915_BO_ALLOC_PM_EARLY))) 51 + return ERR_PTR(-EINVAL); 52 + 48 53 if (!mem) 49 54 return ERR_PTR(-ENODEV); 50 55 ··· 71 66 obj = i915_gem_object_alloc(); 72 67 if (!obj) 73 68 return ERR_PTR(-ENOMEM); 69 + 70 + /* 71 + * Anything smaller than the min_page_size can't be freely inserted into 72 + * the GTT, due to alignemnt restrictions. For such special objects, 73 + * make sure we force memcpy based suspend-resume. In the future we can 74 + * revisit this, either by allowing special mis-aligned objects in the 75 + * migration path, or by mapping all of LMEM upfront using cheap 1G 76 + * GTT entries. 77 + */ 78 + if (default_page_size < mem->min_page_size) 79 + flags |= I915_BO_ALLOC_PM_EARLY; 74 80 75 81 err = mem->ops->init_object(mem, obj, size, page_size, flags); 76 82 if (err)

+1 -1

drivers/gpu/drm/i915/gem/i915_gem_shmem.c

··· 699 699 { 700 700 return intel_memory_region_create(i915, 0, 701 701 totalram_pages() << PAGE_SHIFT, 702 - PAGE_SIZE, 0, 702 + PAGE_SIZE, 0, 0, 703 703 type, instance, 704 704 &shmem_region_ops); 705 705 }

+11 -6

drivers/gpu/drm/i915/gem/i915_gem_stolen.c

··· 492 492 493 493 /* Exclude the reserved region from driver use */ 494 494 mem->region.end = reserved_base - 1; 495 + mem->io_size = resource_size(&mem->region); 495 496 496 497 /* It is possible for the reserved area to end before the end of stolen 497 498 * memory, so just consider the start. */ 498 499 reserved_total = stolen_top - reserved_base; 499 500 501 + i915->stolen_usable_size = 502 + resource_size(&i915->dsm) - reserved_total; 503 + 500 504 drm_dbg(&i915->drm, 501 505 "Memory reserved for graphics device: %lluK, usable: %lluK\n", 502 506 (u64)resource_size(&i915->dsm) >> 10, 503 - ((u64)resource_size(&i915->dsm) - reserved_total) >> 10); 507 + (u64)i915->stolen_usable_size >> 10); 504 508 505 - i915->stolen_usable_size = 506 - resource_size(&i915->dsm) - reserved_total; 509 + if (i915->stolen_usable_size == 0) 510 + return 0; 507 511 508 512 /* Basic memrange allocator for stolen space. */ 509 513 drm_mm_init(&i915->mm.stolen, 0, i915->stolen_usable_size); ··· 752 748 753 749 if (!io_mapping_init_wc(&mem->iomap, 754 750 mem->io_start, 755 - resource_size(&mem->region))) 751 + mem->io_size)) 756 752 return -EIO; 757 753 758 754 /* ··· 807 803 I915_GTT_PAGE_SIZE_4K; 808 804 809 805 mem = intel_memory_region_create(i915, lmem_base, lmem_size, 810 - min_page_size, io_start, 806 + min_page_size, 807 + io_start, lmem_size, 811 808 type, instance, 812 809 &i915_region_stolen_lmem_ops); 813 810 if (IS_ERR(mem)) ··· 839 834 mem = intel_memory_region_create(i915, 840 835 intel_graphics_stolen_res.start, 841 836 resource_size(&intel_graphics_stolen_res), 842 - PAGE_SIZE, 0, type, instance, 837 + PAGE_SIZE, 0, 0, type, instance, 843 838 &i915_region_stolen_smem_ops); 844 839 if (IS_ERR(mem)) 845 840 return mem;

+121 -7

drivers/gpu/drm/i915/gem/i915_gem_ttm.c

··· 7 7 8 8 #include <drm/ttm/ttm_bo_driver.h> 9 9 #include <drm/ttm/ttm_placement.h> 10 + #include <drm/drm_buddy.h> 10 11 11 12 #include "i915_drv.h" 13 + #include "i915_ttm_buddy_manager.h" 12 14 #include "intel_memory_region.h" 13 15 #include "intel_region_ttm.h" 14 16 ··· 24 22 #define I915_TTM_PRIO_PURGE 0 25 23 #define I915_TTM_PRIO_NO_PAGES 1 26 24 #define I915_TTM_PRIO_HAS_PAGES 2 25 + #define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 27 26 28 27 /* 29 28 * Size of struct ttm_place vector in on-stack struct ttm_placement allocs ··· 132 129 place->mem_type = intel_region_to_ttm_type(mr); 133 130 134 131 if (flags & I915_BO_ALLOC_CONTIGUOUS) 135 - place->flags = TTM_PL_FLAG_CONTIGUOUS; 132 + place->flags |= TTM_PL_FLAG_CONTIGUOUS; 133 + if (mr->io_size && mr->io_size < mr->total) { 134 + if (flags & I915_BO_ALLOC_GPU_ONLY) { 135 + place->flags |= TTM_PL_FLAG_TOPDOWN; 136 + } else { 137 + place->fpfn = 0; 138 + place->lpfn = mr->io_size >> PAGE_SHIFT; 139 + } 140 + } 136 141 } 137 142 138 143 static void ··· 342 331 const struct ttm_place *place) 343 332 { 344 333 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 334 + struct ttm_resource *res = bo->resource; 345 335 346 336 if (!obj) 347 337 return false; ··· 356 344 return false; 357 345 358 346 /* Will do for now. Our pinned objects are still on TTM's LRU lists */ 359 - return i915_gem_object_evictable(obj); 347 + if (!i915_gem_object_evictable(obj)) 348 + return false; 349 + 350 + switch (res->mem_type) { 351 + case I915_PL_LMEM0: { 352 + struct ttm_resource_manager *man = 353 + ttm_manager_type(bo->bdev, res->mem_type); 354 + struct i915_ttm_buddy_resource *bman_res = 355 + to_ttm_buddy_resource(res); 356 + struct drm_buddy *mm = bman_res->mm; 357 + struct drm_buddy_block *block; 358 + 359 + if (!place->fpfn && !place->lpfn) 360 + return true; 361 + 362 + GEM_BUG_ON(!place->lpfn); 363 + 364 + /* 365 + * If we just want something mappable then we can quickly check 366 + * if the current victim resource is using any of the CPU 367 + * visible portion. 368 + */ 369 + if (!place->fpfn && 370 + place->lpfn == i915_ttm_buddy_man_visible_size(man)) 371 + return bman_res->used_visible_size > 0; 372 + 373 + /* Real range allocation */ 374 + list_for_each_entry(block, &bman_res->blocks, link) { 375 + unsigned long fpfn = 376 + drm_buddy_block_offset(block) >> PAGE_SHIFT; 377 + unsigned long lpfn = fpfn + 378 + (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 379 + 380 + if (place->fpfn < lpfn && place->lpfn > fpfn) 381 + return true; 382 + } 383 + return false; 384 + } default: 385 + break; 386 + } 387 + 388 + return true; 360 389 } 361 390 362 391 static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, ··· 638 585 i915_ttm_purge(obj); 639 586 } 640 587 588 + static bool i915_ttm_resource_mappable(struct ttm_resource *res) 589 + { 590 + struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 591 + 592 + if (!i915_ttm_cpu_maps_iomem(res)) 593 + return true; 594 + 595 + return bman_res->used_visible_size == bman_res->base.num_pages; 596 + } 597 + 641 598 static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) 642 599 { 643 600 if (!i915_ttm_cpu_maps_iomem(mem)) 644 601 return 0; 602 + 603 + if (!i915_ttm_resource_mappable(mem)) 604 + return -EINVAL; 645 605 646 606 mem->bus.caching = ttm_write_combined; 647 607 mem->bus.is_iomem = true; ··· 794 728 * Gem forced migration using the i915_ttm_migrate() op, is allowed even 795 729 * to regions that are not in the object's list of allowable placements. 796 730 */ 797 - static int i915_ttm_migrate(struct drm_i915_gem_object *obj, 798 - struct intel_memory_region *mr) 731 + static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, 732 + struct intel_memory_region *mr, 733 + unsigned int flags) 799 734 { 800 735 struct ttm_place requested; 801 736 struct ttm_placement placement; 802 737 int ret; 803 738 804 - i915_ttm_place_from_region(mr, &requested, obj->flags); 739 + i915_ttm_place_from_region(mr, &requested, flags); 805 740 placement.num_placement = 1; 806 741 placement.num_busy_placement = 1; 807 742 placement.placement = &requested; ··· 823 756 } 824 757 825 758 return 0; 759 + } 760 + 761 + static int i915_ttm_migrate(struct drm_i915_gem_object *obj, 762 + struct intel_memory_region *mr) 763 + { 764 + return __i915_ttm_migrate(obj, mr, obj->flags); 826 765 } 827 766 828 767 static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, ··· 917 844 } else if (!i915_gem_object_has_pages(obj)) { 918 845 bo->priority = I915_TTM_PRIO_NO_PAGES; 919 846 } else { 920 - bo->priority = I915_TTM_PRIO_HAS_PAGES; 847 + struct ttm_resource_manager *man = 848 + ttm_manager_type(bo->bdev, bo->resource->mem_type); 849 + 850 + /* 851 + * If we need to place an LMEM resource which doesn't need CPU 852 + * access then we should try not to victimize mappable objects 853 + * first, since we likely end up stealing more of the mappable 854 + * portion. And likewise when we try to find space for a mappble 855 + * object, we know not to ever victimize objects that don't 856 + * occupy any mappable pages. 857 + */ 858 + if (i915_ttm_cpu_maps_iomem(bo->resource) && 859 + i915_ttm_buddy_man_visible_size(man) < man->size && 860 + !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) 861 + bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; 862 + else 863 + bo->priority = I915_TTM_PRIO_HAS_PAGES; 921 864 } 922 865 923 866 ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); ··· 987 898 if (obj->mm.madv != I915_MADV_WILLNEED) { 988 899 dma_resv_unlock(bo->base.resv); 989 900 return VM_FAULT_SIGBUS; 901 + } 902 + 903 + if (!i915_ttm_resource_mappable(bo->resource)) { 904 + int err = -ENODEV; 905 + int i; 906 + 907 + for (i = 0; i < obj->mm.n_placements; i++) { 908 + struct intel_memory_region *mr = obj->mm.placements[i]; 909 + unsigned int flags; 910 + 911 + if (!mr->io_size && mr->type != INTEL_MEMORY_SYSTEM) 912 + continue; 913 + 914 + flags = obj->flags; 915 + flags &= ~I915_BO_ALLOC_GPU_ONLY; 916 + err = __i915_ttm_migrate(obj, mr, flags); 917 + if (!err) 918 + break; 919 + } 920 + 921 + if (err) { 922 + drm_dbg(dev, "Unable to make resource CPU accessible\n"); 923 + dma_resv_unlock(bo->base.resv); 924 + return VM_FAULT_SIGBUS; 925 + } 990 926 } 991 927 992 928 if (drm_dev_enter(dev, &idx)) { ··· 1217 1103 1218 1104 mr = intel_memory_region_create(i915, 0, 1219 1105 totalram_pages() << PAGE_SHIFT, 1220 - PAGE_SIZE, 0, 1106 + PAGE_SIZE, 0, 0, 1221 1107 type, instance, 1222 1108 &ttm_system_region_ops); 1223 1109 if (IS_ERR(mr))

+62 -2

drivers/gpu/drm/i915/gem/selftests/huge_pages.c

··· 500 500 int bit; 501 501 int err = 0; 502 502 503 - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 503 + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); 504 504 if (IS_ERR(mem)) { 505 505 pr_err("%s failed to create memory region\n", __func__); 506 506 return PTR_ERR(mem); ··· 1345 1345 1346 1346 err = i915_gem_object_pin_pages_unlocked(obj); 1347 1347 if (err) { 1348 - if (err == -ENXIO || err == -E2BIG) { 1348 + if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) { 1349 1349 i915_gem_object_put(obj); 1350 1350 size >>= 1; 1351 1351 goto try_again; ··· 1473 1473 } 1474 1474 1475 1475 out: 1476 + if (err == -ENOMEM) 1477 + err = 0; 1478 + 1479 + return err; 1480 + } 1481 + 1482 + static int igt_ppgtt_compact(void *arg) 1483 + { 1484 + struct drm_i915_private *i915 = arg; 1485 + struct drm_i915_gem_object *obj; 1486 + int err; 1487 + 1488 + /* 1489 + * Simple test to catch issues with compact 64K pages -- since the pt is 1490 + * compacted to 256B that gives us 32 entries per pt, however since the 1491 + * backing page for the pt is 4K, any extra entries we might incorrectly 1492 + * write out should be ignored by the HW. If ever hit such a case this 1493 + * test should catch it since some of our writes would land in scratch. 1494 + */ 1495 + 1496 + if (!HAS_64K_PAGES(i915)) { 1497 + pr_info("device lacks compact 64K page support, skipping\n"); 1498 + return 0; 1499 + } 1500 + 1501 + if (!HAS_LMEM(i915)) { 1502 + pr_info("device lacks LMEM support, skipping\n"); 1503 + return 0; 1504 + } 1505 + 1506 + /* We want the range to cover multiple page-table boundaries. */ 1507 + obj = i915_gem_object_create_lmem(i915, SZ_4M, 0); 1508 + if (IS_ERR(obj)) 1509 + return PTR_ERR(obj); 1510 + 1511 + err = i915_gem_object_pin_pages_unlocked(obj); 1512 + if (err) 1513 + goto out_put; 1514 + 1515 + if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { 1516 + pr_info("LMEM compact unable to allocate huge-page(s)\n"); 1517 + goto out_unpin; 1518 + } 1519 + 1520 + /* 1521 + * Disable 2M GTT pages by forcing the page-size to 64K for the GTT 1522 + * insertion. 1523 + */ 1524 + obj->mm.page_sizes.sg = I915_GTT_PAGE_SIZE_64K; 1525 + 1526 + err = igt_write_huge(i915, obj); 1527 + if (err) 1528 + pr_err("LMEM compact write-huge failed\n"); 1529 + 1530 + out_unpin: 1531 + i915_gem_object_unpin_pages(obj); 1532 + out_put: 1533 + i915_gem_object_put(obj); 1534 + 1476 1535 if (err == -ENOMEM) 1477 1536 err = 0; 1478 1537 ··· 1795 1736 SUBTEST(igt_tmpfs_fallback), 1796 1737 SUBTEST(igt_ppgtt_smoke_huge), 1797 1738 SUBTEST(igt_ppgtt_sanity_check), 1739 + SUBTEST(igt_ppgtt_compact), 1798 1740 }; 1799 1741 1800 1742 if (!HAS_PPGTT(i915)) {

+13 -8

drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c

··· 40 40 struct blit_buffer scratch; 41 41 struct i915_vma *batch; 42 42 u64 hole; 43 + u64 align; 43 44 u32 width; 44 45 u32 height; 45 46 }; ··· 412 411 goto err_free; 413 412 } 414 413 415 - hole_size = 2 * PAGE_ALIGN(WIDTH * HEIGHT * 4); 414 + t->align = i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_LOCAL); 415 + t->align = max(t->align, 416 + i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_SYSTEM)); 417 + 418 + hole_size = 2 * round_up(WIDTH * HEIGHT * 4, t->align); 416 419 hole_size *= 2; /* room to maneuver */ 417 - hole_size += 2 * I915_GTT_MIN_ALIGNMENT; 420 + hole_size += 2 * t->align; /* padding on either side */ 418 421 419 422 mutex_lock(&t->ce->vm->mutex); 420 423 memset(&hole, 0, sizeof(hole)); 421 424 err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole, 422 - hole_size, 0, I915_COLOR_UNEVICTABLE, 425 + hole_size, t->align, 426 + I915_COLOR_UNEVICTABLE, 423 427 0, U64_MAX, 424 428 DRM_MM_INSERT_BEST); 425 429 if (!err) ··· 435 429 goto err_put; 436 430 } 437 431 438 - t->hole = hole.start + I915_GTT_MIN_ALIGNMENT; 432 + t->hole = hole.start + t->align; 439 433 pr_info("Using hole at %llx\n", t->hole); 440 434 441 435 err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng); ··· 462 456 static int tiled_blits_prepare(struct tiled_blits *t, 463 457 struct rnd_state *prng) 464 458 { 465 - u64 offset = PAGE_ALIGN(t->width * t->height * 4); 459 + u64 offset = round_up(t->width * t->height * 4, t->align); 466 460 u32 *map; 467 461 int err; 468 462 int i; ··· 493 487 494 488 static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng) 495 489 { 496 - u64 offset = 497 - round_up(t->width * t->height * 4, 2 * I915_GTT_MIN_ALIGNMENT); 490 + u64 offset = round_up(t->width * t->height * 4, 2 * t->align); 498 491 int err; 499 492 500 493 /* We want to check position invariant tiling across GTT eviction */ ··· 506 501 507 502 /* Reposition so that we overlap the old addresses, and slightly off */ 508 503 err = tiled_blit(t, 509 - &t->buffers[2], t->hole + I915_GTT_MIN_ALIGNMENT, 504 + &t->buffers[2], t->hole + t->align, 510 505 &t->buffers[1], t->hole + 3 * offset / 2); 511 506 if (err) 512 507 return err;

+5 -3

drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c

··· 885 885 return err; 886 886 } 887 887 888 - static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *vma) 888 + static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, 889 + struct i915_vma *vma, 890 + struct intel_engine_cs *engine) 889 891 { 890 892 u32 *cmd; 891 893 ··· 898 896 return PTR_ERR(cmd); 899 897 900 898 *cmd++ = MI_STORE_REGISTER_MEM_GEN8; 901 - *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE)); 899 + *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(engine->mmio_base)); 902 900 *cmd++ = lower_32_bits(vma->node.start); 903 901 *cmd++ = upper_32_bits(vma->node.start); 904 902 *cmd = MI_BATCH_BUFFER_END; ··· 959 957 if (err) 960 958 goto err_vma; 961 959 962 - err = rpcs_query_batch(rpcs, vma); 960 + err = rpcs_query_batch(rpcs, vma, ce->engine); 963 961 if (err) 964 962 goto err_batch; 965 963

+331 -2

drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c

··· 8 8 9 9 #include "gem/i915_gem_internal.h" 10 10 #include "gem/i915_gem_region.h" 11 + #include "gem/i915_gem_ttm.h" 11 12 #include "gt/intel_engine_pm.h" 12 13 #include "gt/intel_gpu_commands.h" 13 14 #include "gt/intel_gt.h" 14 15 #include "gt/intel_gt_pm.h" 16 + #include "gt/intel_migrate.h" 17 + #include "i915_ttm_buddy_manager.h" 15 18 16 19 #include "huge_gem_object.h" 17 20 #include "i915_selftest.h" ··· 172 169 173 170 out: 174 171 i915_gem_object_lock(obj, NULL); 175 - __i915_vma_put(vma); 172 + i915_vma_destroy(vma); 176 173 i915_gem_object_unlock(obj); 177 174 return err; 178 175 } ··· 269 266 return err; 270 267 271 268 i915_gem_object_lock(obj, NULL); 272 - __i915_vma_put(vma); 269 + i915_vma_destroy(vma); 273 270 i915_gem_object_unlock(obj); 274 271 275 272 if (igt_timeout(end_time, ··· 1004 1001 return 0; 1005 1002 } 1006 1003 1004 + static void igt_close_objects(struct drm_i915_private *i915, 1005 + struct list_head *objects) 1006 + { 1007 + struct drm_i915_gem_object *obj, *on; 1008 + 1009 + list_for_each_entry_safe(obj, on, objects, st_link) { 1010 + i915_gem_object_lock(obj, NULL); 1011 + if (i915_gem_object_has_pinned_pages(obj)) 1012 + i915_gem_object_unpin_pages(obj); 1013 + /* No polluting the memory region between tests */ 1014 + __i915_gem_object_put_pages(obj); 1015 + i915_gem_object_unlock(obj); 1016 + list_del(&obj->st_link); 1017 + i915_gem_object_put(obj); 1018 + } 1019 + 1020 + cond_resched(); 1021 + 1022 + i915_gem_drain_freed_objects(i915); 1023 + } 1024 + 1025 + static void igt_make_evictable(struct list_head *objects) 1026 + { 1027 + struct drm_i915_gem_object *obj; 1028 + 1029 + list_for_each_entry(obj, objects, st_link) { 1030 + i915_gem_object_lock(obj, NULL); 1031 + if (i915_gem_object_has_pinned_pages(obj)) 1032 + i915_gem_object_unpin_pages(obj); 1033 + i915_gem_object_unlock(obj); 1034 + } 1035 + 1036 + cond_resched(); 1037 + } 1038 + 1039 + static int igt_fill_mappable(struct intel_memory_region *mr, 1040 + struct list_head *objects) 1041 + { 1042 + u64 size, total; 1043 + int err; 1044 + 1045 + total = 0; 1046 + size = mr->io_size; 1047 + do { 1048 + struct drm_i915_gem_object *obj; 1049 + 1050 + obj = i915_gem_object_create_region(mr, size, 0, 0); 1051 + if (IS_ERR(obj)) { 1052 + err = PTR_ERR(obj); 1053 + goto err_close; 1054 + } 1055 + 1056 + list_add(&obj->st_link, objects); 1057 + 1058 + err = i915_gem_object_pin_pages_unlocked(obj); 1059 + if (err) { 1060 + if (err != -ENXIO && err != -ENOMEM) 1061 + goto err_close; 1062 + 1063 + if (size == mr->min_page_size) { 1064 + err = 0; 1065 + break; 1066 + } 1067 + 1068 + size >>= 1; 1069 + continue; 1070 + } 1071 + 1072 + total += obj->base.size; 1073 + } while (1); 1074 + 1075 + pr_info("%s filled=%lluMiB\n", __func__, total >> 20); 1076 + return 0; 1077 + 1078 + err_close: 1079 + igt_close_objects(mr->i915, objects); 1080 + return err; 1081 + } 1082 + 1083 + static int ___igt_mmap_migrate(struct drm_i915_private *i915, 1084 + struct drm_i915_gem_object *obj, 1085 + unsigned long addr, 1086 + bool unfaultable) 1087 + { 1088 + struct vm_area_struct *area; 1089 + int err = 0, i; 1090 + 1091 + pr_info("igt_mmap(%s, %d) @ %lx\n", 1092 + obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr); 1093 + 1094 + mmap_read_lock(current->mm); 1095 + area = vma_lookup(current->mm, addr); 1096 + mmap_read_unlock(current->mm); 1097 + if (!area) { 1098 + pr_err("%s: Did not create a vm_area_struct for the mmap\n", 1099 + obj->mm.region->name); 1100 + err = -EINVAL; 1101 + goto out_unmap; 1102 + } 1103 + 1104 + for (i = 0; i < obj->base.size / sizeof(u32); i++) { 1105 + u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); 1106 + u32 x; 1107 + 1108 + if (get_user(x, ux)) { 1109 + err = -EFAULT; 1110 + if (!unfaultable) { 1111 + pr_err("%s: Unable to read from mmap, offset:%zd\n", 1112 + obj->mm.region->name, i * sizeof(x)); 1113 + goto out_unmap; 1114 + } 1115 + 1116 + continue; 1117 + } 1118 + 1119 + if (unfaultable) { 1120 + pr_err("%s: Faulted unmappable memory\n", 1121 + obj->mm.region->name); 1122 + err = -EINVAL; 1123 + goto out_unmap; 1124 + } 1125 + 1126 + if (x != expand32(POISON_INUSE)) { 1127 + pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", 1128 + obj->mm.region->name, 1129 + i * sizeof(x), x, expand32(POISON_INUSE)); 1130 + err = -EINVAL; 1131 + goto out_unmap; 1132 + } 1133 + 1134 + x = expand32(POISON_FREE); 1135 + if (put_user(x, ux)) { 1136 + pr_err("%s: Unable to write to mmap, offset:%zd\n", 1137 + obj->mm.region->name, i * sizeof(x)); 1138 + err = -EFAULT; 1139 + goto out_unmap; 1140 + } 1141 + } 1142 + 1143 + if (unfaultable) { 1144 + if (err == -EFAULT) 1145 + err = 0; 1146 + } else { 1147 + obj->flags &= ~I915_BO_ALLOC_GPU_ONLY; 1148 + err = wc_check(obj); 1149 + } 1150 + out_unmap: 1151 + vm_munmap(addr, obj->base.size); 1152 + return err; 1153 + } 1154 + 1155 + #define IGT_MMAP_MIGRATE_TOPDOWN (1 << 0) 1156 + #define IGT_MMAP_MIGRATE_FILL (1 << 1) 1157 + #define IGT_MMAP_MIGRATE_EVICTABLE (1 << 2) 1158 + #define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3) 1159 + static int __igt_mmap_migrate(struct intel_memory_region **placements, 1160 + int n_placements, 1161 + struct intel_memory_region *expected_mr, 1162 + unsigned int flags) 1163 + { 1164 + struct drm_i915_private *i915 = placements[0]->i915; 1165 + struct drm_i915_gem_object *obj; 1166 + struct i915_request *rq = NULL; 1167 + unsigned long addr; 1168 + LIST_HEAD(objects); 1169 + u64 offset; 1170 + int err; 1171 + 1172 + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, 1173 + placements, 1174 + n_placements); 1175 + if (IS_ERR(obj)) 1176 + return PTR_ERR(obj); 1177 + 1178 + if (flags & IGT_MMAP_MIGRATE_TOPDOWN) 1179 + obj->flags |= I915_BO_ALLOC_GPU_ONLY; 1180 + 1181 + err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL); 1182 + if (err) 1183 + goto out_put; 1184 + 1185 + /* 1186 + * This will eventually create a GEM context, due to opening dummy drm 1187 + * file, which needs a tiny amount of mappable device memory for the top 1188 + * level paging structures(and perhaps scratch), so make sure we 1189 + * allocate early, to avoid tears. 1190 + */ 1191 + addr = igt_mmap_offset(i915, offset, obj->base.size, 1192 + PROT_WRITE, MAP_SHARED); 1193 + if (IS_ERR_VALUE(addr)) { 1194 + err = addr; 1195 + goto out_put; 1196 + } 1197 + 1198 + if (flags & IGT_MMAP_MIGRATE_FILL) { 1199 + err = igt_fill_mappable(placements[0], &objects); 1200 + if (err) 1201 + goto out_put; 1202 + } 1203 + 1204 + err = i915_gem_object_lock(obj, NULL); 1205 + if (err) 1206 + goto out_put; 1207 + 1208 + err = i915_gem_object_pin_pages(obj); 1209 + if (err) { 1210 + i915_gem_object_unlock(obj); 1211 + goto out_put; 1212 + } 1213 + 1214 + err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL, 1215 + obj->mm.pages->sgl, obj->cache_level, 1216 + i915_gem_object_is_lmem(obj), 1217 + expand32(POISON_INUSE), &rq); 1218 + i915_gem_object_unpin_pages(obj); 1219 + if (rq) { 1220 + dma_resv_add_excl_fence(obj->base.resv, &rq->fence); 1221 + i915_gem_object_set_moving_fence(obj, &rq->fence); 1222 + i915_request_put(rq); 1223 + } 1224 + i915_gem_object_unlock(obj); 1225 + if (err) 1226 + goto out_put; 1227 + 1228 + if (flags & IGT_MMAP_MIGRATE_EVICTABLE) 1229 + igt_make_evictable(&objects); 1230 + 1231 + err = ___igt_mmap_migrate(i915, obj, addr, 1232 + flags & IGT_MMAP_MIGRATE_UNFAULTABLE); 1233 + if (!err && obj->mm.region != expected_mr) { 1234 + pr_err("%s region mismatch %s\n", __func__, expected_mr->name); 1235 + err = -EINVAL; 1236 + } 1237 + 1238 + out_put: 1239 + i915_gem_object_put(obj); 1240 + igt_close_objects(i915, &objects); 1241 + return err; 1242 + } 1243 + 1244 + static int igt_mmap_migrate(void *arg) 1245 + { 1246 + struct drm_i915_private *i915 = arg; 1247 + struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM]; 1248 + struct intel_memory_region *mr; 1249 + enum intel_region_id id; 1250 + 1251 + for_each_memory_region(mr, i915, id) { 1252 + struct intel_memory_region *mixed[] = { mr, system }; 1253 + struct intel_memory_region *single[] = { mr }; 1254 + struct ttm_resource_manager *man = mr->region_private; 1255 + resource_size_t saved_io_size; 1256 + int err; 1257 + 1258 + if (mr->private) 1259 + continue; 1260 + 1261 + if (!mr->io_size) 1262 + continue; 1263 + 1264 + /* 1265 + * For testing purposes let's force small BAR, if not already 1266 + * present. 1267 + */ 1268 + saved_io_size = mr->io_size; 1269 + if (mr->io_size == mr->total) { 1270 + resource_size_t io_size = mr->io_size; 1271 + 1272 + io_size = rounddown_pow_of_two(io_size >> 1); 1273 + if (io_size < PAGE_SIZE) 1274 + continue; 1275 + 1276 + mr->io_size = io_size; 1277 + i915_ttm_buddy_man_force_visible_size(man, 1278 + io_size >> PAGE_SHIFT); 1279 + } 1280 + 1281 + /* 1282 + * Allocate in the mappable portion, should be no suprises here. 1283 + */ 1284 + err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0); 1285 + if (err) 1286 + goto out_io_size; 1287 + 1288 + /* 1289 + * Allocate in the non-mappable portion, but force migrating to 1290 + * the mappable portion on fault (LMEM -> LMEM) 1291 + */ 1292 + err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1293 + IGT_MMAP_MIGRATE_TOPDOWN | 1294 + IGT_MMAP_MIGRATE_FILL | 1295 + IGT_MMAP_MIGRATE_EVICTABLE); 1296 + if (err) 1297 + goto out_io_size; 1298 + 1299 + /* 1300 + * Allocate in the non-mappable portion, but force spilling into 1301 + * system memory on fault (LMEM -> SMEM) 1302 + */ 1303 + err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system, 1304 + IGT_MMAP_MIGRATE_TOPDOWN | 1305 + IGT_MMAP_MIGRATE_FILL); 1306 + if (err) 1307 + goto out_io_size; 1308 + 1309 + /* 1310 + * Allocate in the non-mappable portion, but since the mappable 1311 + * portion is already full, and we can't spill to system memory, 1312 + * then we should expect the fault to fail. 1313 + */ 1314 + err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, 1315 + IGT_MMAP_MIGRATE_TOPDOWN | 1316 + IGT_MMAP_MIGRATE_FILL | 1317 + IGT_MMAP_MIGRATE_UNFAULTABLE); 1318 + out_io_size: 1319 + mr->io_size = saved_io_size; 1320 + i915_ttm_buddy_man_force_visible_size(man, 1321 + mr->io_size >> PAGE_SHIFT); 1322 + if (err) 1323 + return err; 1324 + } 1325 + 1326 + return 0; 1327 + } 1328 + 1007 1329 static const char *repr_mmap_type(enum i915_mmap_type type) 1008 1330 { 1009 1331 switch (type) { ··· 1754 1426 SUBTEST(igt_smoke_tiling), 1755 1427 SUBTEST(igt_mmap_offset_exhaustion), 1756 1428 SUBTEST(igt_mmap), 1429 + SUBTEST(igt_mmap_migrate), 1757 1430 SUBTEST(igt_mmap_access), 1758 1431 SUBTEST(igt_mmap_revoke), 1759 1432 SUBTEST(igt_mmap_gpu),

+25 -9

drivers/gpu/drm/i915/gt/gen8_engine_cs.c

··· 201 201 202 202 int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) 203 203 { 204 + struct intel_engine_cs *engine = rq->engine; 205 + 204 206 if (mode & EMIT_FLUSH) { 205 207 u32 flags = 0; 206 208 u32 *cs; ··· 220 218 flags |= PIPE_CONTROL_QW_WRITE; 221 219 222 220 flags |= PIPE_CONTROL_CS_STALL; 221 + 222 + if (engine->class == COMPUTE_CLASS) 223 + flags &= ~PIPE_CONTROL_3D_FLAGS; 223 224 224 225 cs = intel_ring_begin(rq, 6); 225 226 if (IS_ERR(cs)) ··· 250 245 flags |= PIPE_CONTROL_QW_WRITE; 251 246 252 247 flags |= PIPE_CONTROL_CS_STALL; 248 + 249 + if (engine->class == COMPUTE_CLASS) 250 + flags &= ~PIPE_CONTROL_3D_FLAGS; 253 251 254 252 cs = intel_ring_begin(rq, 8 + 4); 255 253 if (IS_ERR(cs)) ··· 626 618 627 619 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) 628 620 { 621 + struct drm_i915_private *i915 = rq->engine->i915; 622 + u32 flags = (PIPE_CONTROL_CS_STALL | 623 + PIPE_CONTROL_TILE_CACHE_FLUSH | 624 + PIPE_CONTROL_FLUSH_L3 | 625 + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 626 + PIPE_CONTROL_DEPTH_CACHE_FLUSH | 627 + PIPE_CONTROL_DC_FLUSH_ENABLE | 628 + PIPE_CONTROL_FLUSH_ENABLE); 629 + 630 + if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) 631 + /* Wa_1409600907 */ 632 + flags |= PIPE_CONTROL_DEPTH_STALL; 633 + 634 + if (rq->engine->class == COMPUTE_CLASS) 635 + flags &= ~PIPE_CONTROL_3D_FLAGS; 636 + 629 637 cs = gen12_emit_ggtt_write_rcs(cs, 630 638 rq->fence.seqno, 631 639 hwsp_offset(rq), 632 640 PIPE_CONTROL0_HDC_PIPELINE_FLUSH, 633 - PIPE_CONTROL_CS_STALL | 634 - PIPE_CONTROL_TILE_CACHE_FLUSH | 635 - PIPE_CONTROL_FLUSH_L3 | 636 - PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 637 - PIPE_CONTROL_DEPTH_CACHE_FLUSH | 638 - /* Wa_1409600907:tgl */ 639 - PIPE_CONTROL_DEPTH_STALL | 640 - PIPE_CONTROL_DC_FLUSH_ENABLE | 641 - PIPE_CONTROL_FLUSH_ENABLE); 641 + flags); 642 642 643 643 return gen12_emit_fini_breadcrumb_tail(rq, cs); 644 644 }

+153 -5

drivers/gpu/drm/i915/gt/gen8_ppgtt.c

··· 233 233 start, end, lvl); 234 234 } else { 235 235 unsigned int count; 236 + unsigned int pte = gen8_pd_index(start, 0); 237 + unsigned int num_ptes; 236 238 u64 *vaddr; 237 239 238 240 count = gen8_pt_count(start, end); ··· 244 242 atomic_read(&pt->used)); 245 243 GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); 246 244 245 + num_ptes = count; 246 + if (pt->is_compact) { 247 + GEM_BUG_ON(num_ptes % 16); 248 + GEM_BUG_ON(pte % 16); 249 + num_ptes /= 16; 250 + pte /= 16; 251 + } 252 + 247 253 vaddr = px_vaddr(pt); 248 - memset64(vaddr + gen8_pd_index(start, 0), 254 + memset64(vaddr + pte, 249 255 vm->scratch[0]->encode, 250 - count); 256 + num_ptes); 251 257 252 258 atomic_sub(count, &pt->used); 253 259 start += count; ··· 463 453 return idx; 464 454 } 465 455 456 + static void 457 + xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, 458 + struct i915_vma_resource *vma_res, 459 + struct sgt_dma *iter, 460 + enum i915_cache_level cache_level, 461 + u32 flags) 462 + { 463 + const gen8_pte_t pte_encode = vm->pte_encode(0, cache_level, flags); 464 + unsigned int rem = sg_dma_len(iter->sg); 465 + u64 start = vma_res->start; 466 + 467 + GEM_BUG_ON(!i915_vm_is_4lvl(vm)); 468 + 469 + do { 470 + struct i915_page_directory * const pdp = 471 + gen8_pdp_for_page_address(vm, start); 472 + struct i915_page_directory * const pd = 473 + i915_pd_entry(pdp, __gen8_pte_index(start, 2)); 474 + struct i915_page_table *pt = 475 + i915_pt_entry(pd, __gen8_pte_index(start, 1)); 476 + gen8_pte_t encode = pte_encode; 477 + unsigned int page_size; 478 + gen8_pte_t *vaddr; 479 + u16 index, max; 480 + 481 + max = I915_PDES; 482 + 483 + if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && 484 + IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && 485 + rem >= I915_GTT_PAGE_SIZE_2M && 486 + !__gen8_pte_index(start, 0)) { 487 + index = __gen8_pte_index(start, 1); 488 + encode |= GEN8_PDE_PS_2M; 489 + page_size = I915_GTT_PAGE_SIZE_2M; 490 + 491 + vaddr = px_vaddr(pd); 492 + } else { 493 + if (encode & GEN12_PPGTT_PTE_LM) { 494 + GEM_BUG_ON(__gen8_pte_index(start, 0) % 16); 495 + GEM_BUG_ON(rem < I915_GTT_PAGE_SIZE_64K); 496 + GEM_BUG_ON(!IS_ALIGNED(iter->dma, 497 + I915_GTT_PAGE_SIZE_64K)); 498 + 499 + index = __gen8_pte_index(start, 0) / 16; 500 + page_size = I915_GTT_PAGE_SIZE_64K; 501 + 502 + max /= 16; 503 + 504 + vaddr = px_vaddr(pd); 505 + vaddr[__gen8_pte_index(start, 1)] |= GEN12_PDE_64K; 506 + 507 + pt->is_compact = true; 508 + } else { 509 + GEM_BUG_ON(pt->is_compact); 510 + index = __gen8_pte_index(start, 0); 511 + page_size = I915_GTT_PAGE_SIZE; 512 + } 513 + 514 + vaddr = px_vaddr(pt); 515 + } 516 + 517 + do { 518 + GEM_BUG_ON(rem < page_size); 519 + vaddr[index++] = encode | iter->dma; 520 + 521 + start += page_size; 522 + iter->dma += page_size; 523 + rem -= page_size; 524 + if (iter->dma >= iter->max) { 525 + iter->sg = __sg_next(iter->sg); 526 + if (!iter->sg) 527 + break; 528 + 529 + rem = sg_dma_len(iter->sg); 530 + if (!rem) 531 + break; 532 + 533 + iter->dma = sg_dma_address(iter->sg); 534 + iter->max = iter->dma + rem; 535 + 536 + if (unlikely(!IS_ALIGNED(iter->dma, page_size))) 537 + break; 538 + } 539 + } while (rem >= page_size && index < max); 540 + 541 + vma_res->page_sizes_gtt |= page_size; 542 + } while (iter->sg && sg_dma_len(iter->sg)); 543 + } 544 + 466 545 static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, 467 546 struct i915_vma_resource *vma_res, 468 547 struct sgt_dma *iter, ··· 685 586 struct sgt_dma iter = sgt_dma(vma_res); 686 587 687 588 if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) { 688 - gen8_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); 589 + if (HAS_64K_PAGES(vm->i915)) 590 + xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); 591 + else 592 + gen8_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); 689 593 } else { 690 594 u64 idx = vma_res->start >> GEN8_PTE_SHIFT; 691 595 ··· 715 613 gen8_pdp_for_page_index(vm, idx); 716 614 struct i915_page_directory *pd = 717 615 i915_pd_entry(pdp, gen8_pd_index(idx, 2)); 616 + struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1)); 718 617 gen8_pte_t *vaddr; 719 618 720 - vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); 619 + GEM_BUG_ON(pt->is_compact); 620 + 621 + vaddr = px_vaddr(pt); 721 622 vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); 722 623 clflush_cache_range(&vaddr[gen8_pd_index(idx, 0)], sizeof(*vaddr)); 624 + } 625 + 626 + static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm, 627 + dma_addr_t addr, 628 + u64 offset, 629 + enum i915_cache_level level, 630 + u32 flags) 631 + { 632 + u64 idx = offset >> GEN8_PTE_SHIFT; 633 + struct i915_page_directory * const pdp = 634 + gen8_pdp_for_page_index(vm, idx); 635 + struct i915_page_directory *pd = 636 + i915_pd_entry(pdp, gen8_pd_index(idx, 2)); 637 + struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1)); 638 + gen8_pte_t *vaddr; 639 + 640 + GEM_BUG_ON(!IS_ALIGNED(addr, SZ_64K)); 641 + GEM_BUG_ON(!IS_ALIGNED(offset, SZ_64K)); 642 + 643 + if (!pt->is_compact) { 644 + vaddr = px_vaddr(pd); 645 + vaddr[gen8_pd_index(idx, 1)] |= GEN12_PDE_64K; 646 + pt->is_compact = true; 647 + } 648 + 649 + vaddr = px_vaddr(pt); 650 + vaddr[gen8_pd_index(idx, 0) / 16] = gen8_pte_encode(addr, level, flags); 651 + } 652 + 653 + static void xehpsdv_ppgtt_insert_entry(struct i915_address_space *vm, 654 + dma_addr_t addr, 655 + u64 offset, 656 + enum i915_cache_level level, 657 + u32 flags) 658 + { 659 + if (flags & PTE_LM) 660 + return __xehpsdv_ppgtt_insert_entry_lm(vm, addr, offset, 661 + level, flags); 662 + 663 + return gen8_ppgtt_insert_entry(vm, addr, offset, level, flags); 723 664 } 724 665 725 666 static int gen8_init_scratch(struct i915_address_space *vm) ··· 964 819 965 820 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; 966 821 ppgtt->vm.insert_entries = gen8_ppgtt_insert; 967 - ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; 822 + if (HAS_64K_PAGES(gt->i915)) 823 + ppgtt->vm.insert_page = xehpsdv_ppgtt_insert_entry; 824 + else 825 + ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; 968 826 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; 969 827 ppgtt->vm.clear_range = gen8_ppgtt_clear; 970 828 ppgtt->vm.foreach = gen8_ppgtt_foreach;

+2

drivers/gpu/drm/i915/gt/intel_engine.h

··· 265 265 266 266 void intel_engine_destroy_pinned_context(struct intel_context *ce); 267 267 268 + void xehp_enable_ccs_engines(struct intel_engine_cs *engine); 269 + 268 270 #define ENGINE_PHYSICAL 0 269 271 #define ENGINE_MOCK 1 270 272 #define ENGINE_VIRTUAL 2

+125 -32

drivers/gpu/drm/i915/gt/intel_engine_cs.c

··· 21 21 #include "intel_gt.h" 22 22 #include "intel_gt_requests.h" 23 23 #include "intel_gt_pm.h" 24 + #include "intel_lrc.h" 24 25 #include "intel_lrc_reg.h" 25 26 #include "intel_reset.h" 26 27 #include "intel_ring.h" ··· 157 156 { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE } 158 157 }, 159 158 }, 159 + [CCS0] = { 160 + .class = COMPUTE_CLASS, 161 + .instance = 0, 162 + .mmio_bases = { 163 + { .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE } 164 + } 165 + }, 166 + [CCS1] = { 167 + .class = COMPUTE_CLASS, 168 + .instance = 1, 169 + .mmio_bases = { 170 + { .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE } 171 + } 172 + }, 173 + [CCS2] = { 174 + .class = COMPUTE_CLASS, 175 + .instance = 2, 176 + .mmio_bases = { 177 + { .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE } 178 + } 179 + }, 180 + [CCS3] = { 181 + .class = COMPUTE_CLASS, 182 + .instance = 3, 183 + .mmio_bases = { 184 + { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE } 185 + } 186 + }, 160 187 }; 161 188 162 189 /** ··· 209 180 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE); 210 181 211 182 switch (class) { 183 + case COMPUTE_CLASS: 184 + fallthrough; 212 185 case RENDER_CLASS: 213 186 switch (GRAPHICS_VER(gt->i915)) { 214 187 default: ··· 324 293 GEM_DEBUG_WARN_ON(iir); 325 294 } 326 295 296 + static u32 get_reset_domain(u8 ver, enum intel_engine_id id) 297 + { 298 + u32 reset_domain; 299 + 300 + if (ver >= 11) { 301 + static const u32 engine_reset_domains[] = { 302 + [RCS0] = GEN11_GRDOM_RENDER, 303 + [BCS0] = GEN11_GRDOM_BLT, 304 + [VCS0] = GEN11_GRDOM_MEDIA, 305 + [VCS1] = GEN11_GRDOM_MEDIA2, 306 + [VCS2] = GEN11_GRDOM_MEDIA3, 307 + [VCS3] = GEN11_GRDOM_MEDIA4, 308 + [VCS4] = GEN11_GRDOM_MEDIA5, 309 + [VCS5] = GEN11_GRDOM_MEDIA6, 310 + [VCS6] = GEN11_GRDOM_MEDIA7, 311 + [VCS7] = GEN11_GRDOM_MEDIA8, 312 + [VECS0] = GEN11_GRDOM_VECS, 313 + [VECS1] = GEN11_GRDOM_VECS2, 314 + [VECS2] = GEN11_GRDOM_VECS3, 315 + [VECS3] = GEN11_GRDOM_VECS4, 316 + [CCS0] = GEN11_GRDOM_RENDER, 317 + [CCS1] = GEN11_GRDOM_RENDER, 318 + [CCS2] = GEN11_GRDOM_RENDER, 319 + [CCS3] = GEN11_GRDOM_RENDER, 320 + }; 321 + GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || 322 + !engine_reset_domains[id]); 323 + reset_domain = engine_reset_domains[id]; 324 + } else { 325 + static const u32 engine_reset_domains[] = { 326 + [RCS0] = GEN6_GRDOM_RENDER, 327 + [BCS0] = GEN6_GRDOM_BLT, 328 + [VCS0] = GEN6_GRDOM_MEDIA, 329 + [VCS1] = GEN8_GRDOM_MEDIA2, 330 + [VECS0] = GEN6_GRDOM_VECS, 331 + }; 332 + GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || 333 + !engine_reset_domains[id]); 334 + reset_domain = engine_reset_domains[id]; 335 + } 336 + 337 + return reset_domain; 338 + } 339 + 327 340 static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, 328 341 u8 logical_instance) 329 342 { ··· 403 328 engine->id = id; 404 329 engine->legacy_idx = INVALID_ENGINE; 405 330 engine->mask = BIT(id); 406 - if (GRAPHICS_VER(gt->i915) >= 11) { 407 - static const u32 engine_reset_domains[] = { 408 - [RCS0] = GEN11_GRDOM_RENDER, 409 - [BCS0] = GEN11_GRDOM_BLT, 410 - [VCS0] = GEN11_GRDOM_MEDIA, 411 - [VCS1] = GEN11_GRDOM_MEDIA2, 412 - [VCS2] = GEN11_GRDOM_MEDIA3, 413 - [VCS3] = GEN11_GRDOM_MEDIA4, 414 - [VCS4] = GEN11_GRDOM_MEDIA5, 415 - [VCS5] = GEN11_GRDOM_MEDIA6, 416 - [VCS6] = GEN11_GRDOM_MEDIA7, 417 - [VCS7] = GEN11_GRDOM_MEDIA8, 418 - [VECS0] = GEN11_GRDOM_VECS, 419 - [VECS1] = GEN11_GRDOM_VECS2, 420 - [VECS2] = GEN11_GRDOM_VECS3, 421 - [VECS3] = GEN11_GRDOM_VECS4, 422 - }; 423 - GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || 424 - !engine_reset_domains[id]); 425 - engine->reset_domain = engine_reset_domains[id]; 426 - } else { 427 - static const u32 engine_reset_domains[] = { 428 - [RCS0] = GEN6_GRDOM_RENDER, 429 - [BCS0] = GEN6_GRDOM_BLT, 430 - [VCS0] = GEN6_GRDOM_MEDIA, 431 - [VCS1] = GEN8_GRDOM_MEDIA2, 432 - [VECS0] = GEN6_GRDOM_VECS, 433 - }; 434 - GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || 435 - !engine_reset_domains[id]); 436 - engine->reset_domain = engine_reset_domains[id]; 437 - } 331 + engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915), 332 + id); 438 333 engine->i915 = i915; 439 334 engine->gt = gt; 440 335 engine->uncore = gt->uncore; ··· 433 388 /* Override to uninterruptible for OpenCL workloads. */ 434 389 if (GRAPHICS_VER(i915) == 12 && engine->class == RENDER_CLASS) 435 390 engine->props.preempt_timeout_ms = 0; 391 + 392 + /* features common between engines sharing EUs */ 393 + if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { 394 + engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; 395 + engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; 396 + } 436 397 437 398 engine->defaults = engine->props; /* never to change again */ 438 399 ··· 592 541 return false; 593 542 } 594 543 544 + static void engine_mask_apply_compute_fuses(struct intel_gt *gt) 545 + { 546 + struct drm_i915_private *i915 = gt->i915; 547 + struct intel_gt_info *info = &gt->info; 548 + int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS; 549 + unsigned long ccs_mask; 550 + unsigned int i; 551 + 552 + if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) 553 + return; 554 + 555 + ccs_mask = intel_slicemask_from_dssmask(intel_sseu_get_compute_subslices(&info->sseu), 556 + ss_per_ccs); 557 + /* 558 + * If all DSS in a quadrant are fused off, the corresponding CCS 559 + * engine is not available for use. 560 + */ 561 + for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) { 562 + info->engine_mask &= ~BIT(_CCS(i)); 563 + drm_dbg(&i915->drm, "ccs%u fused off\n", i); 564 + } 565 + } 566 + 595 567 /* 596 568 * Determine which engines are fused off in our particular hardware. 597 569 * Note that we have a catch-22 situation where we need to be able to access ··· 695 621 drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n", 696 622 vebox_mask, VEBOX_MASK(gt)); 697 623 GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); 624 + 625 + engine_mask_apply_compute_fuses(gt); 698 626 699 627 return info->engine_mask; 700 628 } ··· 2093 2017 } 2094 2018 2095 2019 return active; 2020 + } 2021 + 2022 + void xehp_enable_ccs_engines(struct intel_engine_cs *engine) 2023 + { 2024 + /* 2025 + * If there are any non-fused-off CCS engines, we need to enable CCS 2026 + * support in the RCU_MODE register. This only needs to be done once, 2027 + * so for simplicity we'll take care of this in the RCS engine's 2028 + * resume handler; since the RCS and all CCS engines belong to the 2029 + * same reset domain and are reset together, this will also take care 2030 + * of re-applying the setting after i915-triggered resets. 2031 + */ 2032 + if (!CCS_MASK(engine->gt)) 2033 + return; 2034 + 2035 + intel_uncore_write(engine->uncore, GEN12_RCU_MODE, 2036 + _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE)); 2096 2037 } 2097 2038 2098 2039 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)

+10 -1

drivers/gpu/drm/i915/gt/intel_engine_types.h

··· 33 33 #define VIDEO_ENHANCEMENT_CLASS 2 34 34 #define COPY_ENGINE_CLASS 3 35 35 #define OTHER_CLASS 4 36 - #define MAX_ENGINE_CLASS 4 36 + #define COMPUTE_CLASS 5 37 + #define MAX_ENGINE_CLASS 5 37 38 #define MAX_ENGINE_INSTANCE 7 38 39 39 40 #define I915_MAX_SLICES 3 ··· 96 95 97 96 #define I915_MAX_VCS 8 98 97 #define I915_MAX_VECS 4 98 + #define I915_MAX_CCS 4 99 99 100 100 /* 101 101 * Engine IDs definitions. ··· 119 117 VECS2, 120 118 VECS3, 121 119 #define _VECS(n) (VECS0 + (n)) 120 + CCS0, 121 + CCS1, 122 + CCS2, 123 + CCS3, 124 + #define _CCS(n) (CCS0 + (n)) 122 125 I915_NUM_ENGINES 123 126 #define INVALID_ENGINE ((enum intel_engine_id)-1) 124 127 }; ··· 524 517 #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) 525 518 #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) 526 519 #define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8) 520 + #define I915_ENGINE_HAS_RCS_REG_STATE BIT(9) 521 + #define I915_ENGINE_HAS_EU_PRIORITY BIT(10) 527 522 unsigned int flags; 528 523 529 524 /*

+4 -1

drivers/gpu/drm/i915/gt/intel_engine_user.c

··· 47 47 [COPY_ENGINE_CLASS] = I915_ENGINE_CLASS_COPY, 48 48 [VIDEO_DECODE_CLASS] = I915_ENGINE_CLASS_VIDEO, 49 49 [VIDEO_ENHANCEMENT_CLASS] = I915_ENGINE_CLASS_VIDEO_ENHANCE, 50 + /* TODO: Add COMPUTE_CLASS mapping once ABI is available */ 50 51 }; 51 52 52 53 static int engine_cmp(void *priv, const struct list_head *A, ··· 140 139 [COPY_ENGINE_CLASS] = "bcs", 141 140 [VIDEO_DECODE_CLASS] = "vcs", 142 141 [VIDEO_ENHANCEMENT_CLASS] = "vecs", 142 + [COMPUTE_CLASS] = "ccs", 143 143 }; 144 144 145 145 if (class >= ARRAY_SIZE(uabi_names) || !uabi_names[class]) ··· 164 162 [COPY_ENGINE_CLASS] = { BCS0, 1 }, 165 163 [VIDEO_DECODE_CLASS] = { VCS0, I915_MAX_VCS }, 166 164 [VIDEO_ENHANCEMENT_CLASS] = { VECS0, I915_MAX_VECS }, 165 + [COMPUTE_CLASS] = { CCS0, I915_MAX_CCS }, 167 166 }; 168 167 169 168 if (GEM_DEBUG_WARN_ON(ring->class >= ARRAY_SIZE(map))) ··· 193 190 void intel_engines_driver_register(struct drm_i915_private *i915) 194 191 { 195 192 struct legacy_ring ring = {}; 196 - u8 uabi_instances[4] = {}; 193 + u8 uabi_instances[5] = {}; 197 194 struct list_head *it, *next; 198 195 struct rb_node **p, *prev; 199 196 LIST_HEAD(engines);

+9 -2

drivers/gpu/drm/i915/gt/intel_execlists_submission.c

··· 665 665 static u64 execlists_update_context(struct i915_request *rq) 666 666 { 667 667 struct intel_context *ce = rq->context; 668 - u64 desc = ce->lrc.desc; 668 + u64 desc; 669 669 u32 tail, prev; 670 + 671 + desc = ce->lrc.desc; 672 + if (rq->engine->flags & I915_ENGINE_HAS_EU_PRIORITY) 673 + desc |= lrc_desc_priority(rq_prio(rq)); 670 674 671 675 /* 672 676 * WaIdleLiteRestore:bdw,skl ··· 2911 2907 2912 2908 enable_execlists(engine); 2913 2909 2910 + if (engine->class == RENDER_CLASS) 2911 + xehp_enable_ccs_engines(engine); 2912 + 2914 2913 return 0; 2915 2914 } 2916 2915 ··· 3487 3480 logical_ring_default_vfuncs(engine); 3488 3481 logical_ring_default_irqs(engine); 3489 3482 3490 - if (engine->class == RENDER_CLASS) 3483 + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) 3491 3484 rcs_submission_override(engine); 3492 3485 3493 3486 lrc_init_wa_ctx(engine);

+15

drivers/gpu/drm/i915/gt/intel_gpu_commands.h

··· 228 228 #define PIPE_CONTROL_COMMAND_CACHE_INVALIDATE (1<<29) /* gen11+ */ 229 229 #define PIPE_CONTROL_TILE_CACHE_FLUSH (1<<28) /* gen11+ */ 230 230 #define PIPE_CONTROL_FLUSH_L3 (1<<27) 231 + #define PIPE_CONTROL_AMFS_FLUSH (1<<25) /* gen12+ */ 231 232 #define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) /* gen7+ */ 232 233 #define PIPE_CONTROL_MMIO_WRITE (1<<23) 233 234 #define PIPE_CONTROL_STORE_DATA_INDEX (1<<21) 234 235 #define PIPE_CONTROL_CS_STALL (1<<20) 236 + #define PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET (1<<19) 235 237 #define PIPE_CONTROL_TLB_INVALIDATE (1<<18) 238 + #define PIPE_CONTROL_PSD_SYNC (1<<17) /* gen11+ */ 236 239 #define PIPE_CONTROL_MEDIA_STATE_CLEAR (1<<16) 237 240 #define PIPE_CONTROL_WRITE_TIMESTAMP (3<<14) 238 241 #define PIPE_CONTROL_QW_WRITE (1<<14) ··· 256 253 #define PIPE_CONTROL_STALL_AT_SCOREBOARD (1<<1) 257 254 #define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1<<0) 258 255 #define PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */ 256 + 257 + /* 3D-related flags can't be set on compute engine */ 258 + #define PIPE_CONTROL_3D_FLAGS (\ 259 + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 260 + PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 261 + PIPE_CONTROL_TILE_CACHE_FLUSH | \ 262 + PIPE_CONTROL_DEPTH_STALL | \ 263 + PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 264 + PIPE_CONTROL_PSD_SYNC | \ 265 + PIPE_CONTROL_AMFS_FLUSH | \ 266 + PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 267 + PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 259 268 260 269 #define MI_MATH(x) MI_INSTR(0x1a, (x) - 1) 261 270 #define MI_MATH_INSTR(opcode, op1, op2) ((opcode) << 20 | (op1) << 10 | (op2))

+22 -3

drivers/gpu/drm/i915/gt/intel_gt.c

··· 65 65 int err; 66 66 67 67 mem = intel_gt_setup_lmem(gt); 68 - if (mem == ERR_PTR(-ENODEV)) 69 - mem = intel_gt_setup_fake_lmem(gt); 70 68 if (IS_ERR(mem)) { 71 69 err = PTR_ERR(mem); 72 70 if (err == -ENODEV) ··· 456 458 struct i915_vma *vma; 457 459 int ret; 458 460 459 - obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE); 461 + obj = i915_gem_object_create_lmem(i915, size, 462 + I915_BO_ALLOC_VOLATILE | 463 + I915_BO_ALLOC_GPU_ONLY); 460 464 if (IS_ERR(obj)) 461 465 obj = i915_gem_object_create_stolen(i915, size); 462 466 if (IS_ERR(obj)) ··· 911 911 } 912 912 913 913 return intel_uncore_read_fw(gt->uncore, reg); 914 + } 915 + 916 + u32 intel_gt_read_register(struct intel_gt *gt, i915_reg_t reg) 917 + { 918 + int type; 919 + u8 sliceid, subsliceid; 920 + 921 + for (type = 0; type < NUM_STEERING_TYPES; type++) { 922 + if (intel_gt_reg_needs_read_steering(gt, reg, type)) { 923 + intel_gt_get_valid_steering(gt, type, &sliceid, 924 + &subsliceid); 925 + return intel_uncore_read_with_mcr_steering(gt->uncore, 926 + reg, 927 + sliceid, 928 + subsliceid); 929 + } 930 + } 931 + 932 + return intel_uncore_read(gt->uncore, reg); 914 933 } 915 934 916 935 void intel_gt_info_print(const struct intel_gt_info *info,

+1

drivers/gpu/drm/i915/gt/intel_gt.h

··· 85 85 } 86 86 87 87 u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg); 88 + u32 intel_gt_read_register(struct intel_gt *gt, i915_reg_t reg); 88 89 89 90 void intel_gt_info_print(const struct intel_gt_info *info, 90 91 struct drm_printer *p);

+14 -1

drivers/gpu/drm/i915/gt/intel_gt_irq.c

··· 100 100 if (unlikely(!intr)) 101 101 return; 102 102 103 - if (class <= COPY_ENGINE_CLASS) 103 + if (class <= COPY_ENGINE_CLASS || class == COMPUTE_CLASS) 104 104 return gen11_engine_irq_handler(gt, class, instance, intr); 105 105 106 106 if (class == OTHER_CLASS) ··· 182 182 /* Disable RCS, BCS, VCS and VECS class engines. */ 183 183 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0); 184 184 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0); 185 + if (CCS_MASK(gt)) 186 + intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0); 185 187 186 188 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ 187 189 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0); ··· 197 195 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0); 198 196 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) 199 197 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0); 198 + if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1)) 199 + intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0); 200 + if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) 201 + intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0); 200 202 201 203 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); 202 204 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); ··· 231 225 /* Enable RCS, BCS, VCS and VECS class interrupts. */ 232 226 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask); 233 227 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask); 228 + if (CCS_MASK(gt)) 229 + intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask); 234 230 235 231 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ 236 232 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); ··· 246 238 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask); 247 239 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) 248 240 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask); 241 + if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1)) 242 + intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask); 243 + if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) 244 + intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask); 245 + 249 246 /* 250 247 * RPS interrupts will get enabled/disabled on demand when RPS itself 251 248 * is enabled/disabled.

+14 -34

drivers/gpu/drm/i915/gt/intel_gt_regs.h

··· 465 465 #define GEN9_PGCTL_SSB_EU210_ACK (1 << 12) 466 466 #define GEN9_PGCTL_SSB_EU311_ACK (1 << 14) 467 467 468 + #define VF_PREEMPTION _MMIO(0x83a4) 469 + #define PREEMPTION_VERTEX_COUNT REG_GENMASK(15, 0) 470 + 468 471 #define GEN8_RC6_CTX_INFO _MMIO(0x8504) 469 472 470 473 #define GEN12_SQCM _MMIO(0x8724) ··· 1060 1057 #define FLOW_CONTROL_ENABLE REG_BIT(15) 1061 1058 #define UGM_BACKUP_MODE REG_BIT(13) 1062 1059 #define MDQ_ARBITRATION_MODE REG_BIT(12) 1060 + #define SYSTOLIC_DOP_CLOCK_GATING_DIS REG_BIT(10) 1063 1061 #define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE REG_BIT(8) 1064 1062 #define STALL_DOP_GATING_DISABLE REG_BIT(5) 1065 1063 #define THROTTLE_12_5 REG_GENMASK(4, 2) ··· 1328 1324 #define ECOBITS_PPGTT_CACHE64B (3 << 8) 1329 1325 #define ECOBITS_PPGTT_CACHE4B (0 << 8) 1330 1326 1327 + #define GEN12_RCU_MODE _MMIO(0x14800) 1328 + #define GEN12_RCU_MODE_CCS_ENABLE REG_BIT(0) 1329 + 1331 1330 #define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) 1332 1331 #define CHV_FGT_DISABLE_SS0 (1 << 10) 1333 1332 #define CHV_FGT_DISABLE_SS1 (1 << 11) ··· 1456 1449 #define GEN11_KCR (19) 1457 1450 #define GEN11_GTPM (16) 1458 1451 #define GEN11_BCS (15) 1452 + #define GEN12_CCS3 (7) 1453 + #define GEN12_CCS2 (6) 1454 + #define GEN12_CCS1 (5) 1455 + #define GEN12_CCS0 (4) 1459 1456 #define GEN11_RCS0 (0) 1460 1457 #define GEN11_VECS(x) (31 - (x)) 1461 1458 #define GEN11_VCS(x) (x) ··· 1472 1461 #define GEN11_GPM_WGBOXPERF_INTR_ENABLE _MMIO(0x19003c) 1473 1462 #define GEN11_CRYPTO_RSVD_INTR_ENABLE _MMIO(0x190040) 1474 1463 #define GEN11_GUNIT_CSME_INTR_ENABLE _MMIO(0x190044) 1464 + #define GEN12_CCS_RSVD_INTR_ENABLE _MMIO(0x190048) 1475 1465 1476 1466 #define GEN11_INTR_IDENTITY_REG(x) _MMIO(0x190060 + ((x) * 4)) 1477 1467 #define GEN11_INTR_DATA_VALID (1 << 31) ··· 1498 1486 #define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) 1499 1487 #define GEN11_CRYPTO_RSVD_INTR_MASK _MMIO(0x1900f0) 1500 1488 #define GEN11_GUNIT_CSME_INTR_MASK _MMIO(0x1900f4) 1489 + #define GEN12_CCS0_CCS1_INTR_MASK _MMIO(0x190100) 1490 + #define GEN12_CCS2_CCS3_INTR_MASK _MMIO(0x190104) 1501 1491 1502 1492 #define GEN12_SFC_DONE(n) _MMIO(0x1cc000 + (n) * 0x1000) 1503 - 1504 - enum { 1505 - INTEL_ADVANCED_CONTEXT = 0, 1506 - INTEL_LEGACY_32B_CONTEXT, 1507 - INTEL_ADVANCED_AD_CONTEXT, 1508 - INTEL_LEGACY_64B_CONTEXT 1509 - }; 1510 - 1511 - enum { 1512 - FAULT_AND_HANG = 0, 1513 - FAULT_AND_HALT, /* Debug only */ 1514 - FAULT_AND_STREAM, 1515 - FAULT_AND_CONTINUE /* Unsupported */ 1516 - }; 1517 - 1518 - #define CTX_GTT_ADDRESS_MASK GENMASK(31, 12) 1519 - #define GEN8_CTX_VALID (1 << 0) 1520 - #define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) 1521 - #define GEN8_CTX_FORCE_RESTORE (1 << 2) 1522 - #define GEN8_CTX_L3LLC_COHERENT (1 << 5) 1523 - #define GEN8_CTX_PRIVILEGE (1 << 8) 1524 - #define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 1525 - #define GEN8_CTX_ID_SHIFT 32 1526 - #define GEN8_CTX_ID_WIDTH 21 1527 - #define GEN11_SW_CTX_ID_SHIFT 37 1528 - #define GEN11_SW_CTX_ID_WIDTH 11 1529 - #define GEN11_ENGINE_CLASS_SHIFT 61 1530 - #define GEN11_ENGINE_CLASS_WIDTH 3 1531 - #define GEN11_ENGINE_INSTANCE_SHIFT 48 1532 - #define GEN11_ENGINE_INSTANCE_WIDTH 6 1533 - #define XEHP_SW_CTX_ID_SHIFT 39 1534 - #define XEHP_SW_CTX_ID_WIDTH 16 1535 - #define XEHP_SW_COUNTER_SHIFT 58 1536 - #define XEHP_SW_COUNTER_WIDTH 6 1537 1493 1538 1494 #endif /* __INTEL_GT_REGS__ */

+23 -6

drivers/gpu/drm/i915/gt/intel_gtt.c

··· 107 107 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 108 108 struct drm_i915_gem_object *obj = vma->obj; 109 109 110 - /* Keep the obj (and hence the vma) alive as _we_ destroy it */ 111 - if (!kref_get_unless_zero(&obj->base.refcount)) 110 + if (!kref_get_unless_zero(&obj->base.refcount)) { 111 + /* 112 + * Unbind the dying vma to ensure the bound_list 113 + * is completely drained. We leave the destruction to 114 + * the object destructor. 115 + */ 116 + atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 117 + WARN_ON(__i915_vma_unbind(vma)); 112 118 continue; 119 + } 113 120 114 - atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 115 - WARN_ON(__i915_vma_unbind(vma)); 116 - __i915_vma_put(vma); 117 - 121 + /* Keep the obj (and hence the vma) alive as _we_ destroy it */ 122 + i915_vma_destroy_locked(vma); 118 123 i915_gem_object_put(obj); 119 124 } 120 125 GEM_BUG_ON(!list_empty(&vm->bound_list)); ··· 230 225 231 226 GEM_BUG_ON(!vm->total); 232 227 drm_mm_init(&vm->mm, 0, vm->total); 228 + 229 + memset64(vm->min_alignment, I915_GTT_MIN_ALIGNMENT, 230 + ARRAY_SIZE(vm->min_alignment)); 231 + 232 + if (HAS_64K_PAGES(vm->i915) && NEEDS_COMPACT_PT(vm->i915)) { 233 + vm->min_alignment[INTEL_MEMORY_LOCAL] = I915_GTT_PAGE_SIZE_2M; 234 + vm->min_alignment[INTEL_MEMORY_STOLEN_LOCAL] = I915_GTT_PAGE_SIZE_2M; 235 + } else if (HAS_64K_PAGES(vm->i915)) { 236 + vm->min_alignment[INTEL_MEMORY_LOCAL] = I915_GTT_PAGE_SIZE_64K; 237 + vm->min_alignment[INTEL_MEMORY_STOLEN_LOCAL] = I915_GTT_PAGE_SIZE_64K; 238 + } 239 + 233 240 vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; 234 241 235 242 INIT_LIST_HEAD(&vm->bound_list);

+34 -1

drivers/gpu/drm/i915/gt/intel_gtt.h

··· 29 29 #include "i915_selftest.h" 30 30 #include "i915_vma_resource.h" 31 31 #include "i915_vma_types.h" 32 + #include "i915_params.h" 33 + #include "intel_memory_region.h" 32 34 33 35 #define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) 34 36 ··· 91 89 #define GEN12_PPGTT_PTE_LM BIT_ULL(11) 92 90 93 91 #define GEN12_GGTT_PTE_LM BIT_ULL(1) 92 + 93 + #define GEN12_PDE_64K BIT(6) 94 94 95 95 /* 96 96 * Cacheability Control is a 4-bit value. The low three bits are stored in bits ··· 162 158 atomic_t used; 163 159 struct i915_page_table *stash; 164 160 }; 161 + bool is_compact; 165 162 }; 166 163 167 164 struct i915_page_directory { ··· 200 195 struct i915_vm_pt_stash { 201 196 /* preallocated chains of page tables/directories */ 202 197 struct i915_page_table *pt[2]; 198 + /* 199 + * Optionally override the alignment/size of the physical page that 200 + * contains each PT. If not set defaults back to the usual 201 + * I915_GTT_PAGE_SIZE_4K. This does not influence the other paging 202 + * structures. MUST be a power-of-two. ONLY applicable on discrete 203 + * platforms. 204 + */ 205 + int pt_sz; 203 206 }; 204 207 205 208 struct i915_vma_ops { ··· 236 223 struct device *dma; 237 224 u64 total; /* size addr space maps (ex. 2GB for ggtt) */ 238 225 u64 reserved; /* size addr space reserved */ 226 + u64 min_alignment[INTEL_MEMORY_STOLEN_LOCAL + 1]; 239 227 240 228 unsigned int bind_async_flags; 241 229 ··· 396 382 i915_vm_has_scratch_64K(struct i915_address_space *vm) 397 383 { 398 384 return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K); 385 + } 386 + 387 + static inline u64 i915_vm_min_alignment(struct i915_address_space *vm, 388 + enum intel_memory_type type) 389 + { 390 + /* avoid INTEL_MEMORY_MOCK overflow */ 391 + if ((int)type >= ARRAY_SIZE(vm->min_alignment)) 392 + type = INTEL_MEMORY_SYSTEM; 393 + 394 + return vm->min_alignment[type]; 395 + } 396 + 397 + static inline u64 i915_vm_obj_min_alignment(struct i915_address_space *vm, 398 + struct drm_i915_gem_object *obj) 399 + { 400 + struct intel_memory_region *mr = READ_ONCE(obj->mm.region); 401 + enum intel_memory_type type = mr ? mr->type : INTEL_MEMORY_SYSTEM; 402 + 403 + return i915_vm_min_alignment(vm, type); 399 404 } 400 405 401 406 static inline bool ··· 603 570 604 571 struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); 605 572 struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz); 606 - struct i915_page_table *alloc_pt(struct i915_address_space *vm); 573 + struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz); 607 574 struct i915_page_directory *alloc_pd(struct i915_address_space *vm); 608 575 struct i915_page_directory *__alloc_pd(int npde); 609 576

+10 -2

drivers/gpu/drm/i915/gt/intel_lrc.c

··· 623 623 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && 624 624 !intel_engine_has_relative_mmio(engine)); 625 625 626 - if (engine->class == RENDER_CLASS) { 626 + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { 627 627 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) 628 628 return dg2_rcs_offsets; 629 629 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) ··· 1217 1217 cs = gen12_emit_timestamp_wa(ce, cs); 1218 1218 cs = gen12_emit_restore_scratch(ce, cs); 1219 1219 1220 + /* Wa_16013000631:dg2 */ 1221 + if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) || 1222 + IS_DG2_G11(ce->engine->i915)) 1223 + if (ce->engine->class == COMPUTE_CLASS) 1224 + cs = gen8_emit_pipe_control(cs, 1225 + PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 1226 + 0); 1227 + 1220 1228 return cs; 1221 1229 } 1222 1230 ··· 1627 1619 unsigned int i; 1628 1620 int err; 1629 1621 1630 - if (engine->class != RENDER_CLASS) 1622 + if (!(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) 1631 1623 return; 1632 1624 1633 1625 switch (GRAPHICS_VER(engine->i915)) {

+51

drivers/gpu/drm/i915/gt/intel_lrc.h

··· 6 6 #ifndef __INTEL_LRC_H__ 7 7 #define __INTEL_LRC_H__ 8 8 9 + #include "i915_priolist_types.h" 10 + 11 + #include <linux/bitfield.h> 9 12 #include <linux/types.h> 10 13 11 14 struct drm_i915_gem_object; ··· 71 68 const char *when); 72 69 73 70 void lrc_update_runtime(struct intel_context *ce); 71 + 72 + enum { 73 + INTEL_ADVANCED_CONTEXT = 0, 74 + INTEL_LEGACY_32B_CONTEXT, 75 + INTEL_ADVANCED_AD_CONTEXT, 76 + INTEL_LEGACY_64B_CONTEXT 77 + }; 78 + 79 + enum { 80 + FAULT_AND_HANG = 0, 81 + FAULT_AND_HALT, /* Debug only */ 82 + FAULT_AND_STREAM, 83 + FAULT_AND_CONTINUE /* Unsupported */ 84 + }; 85 + 86 + #define CTX_GTT_ADDRESS_MASK GENMASK(31, 12) 87 + #define GEN8_CTX_VALID (1 << 0) 88 + #define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) 89 + #define GEN8_CTX_FORCE_RESTORE (1 << 2) 90 + #define GEN8_CTX_L3LLC_COHERENT (1 << 5) 91 + #define GEN8_CTX_PRIVILEGE (1 << 8) 92 + #define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 93 + #define GEN12_CTX_PRIORITY_MASK GENMASK(10, 9) 94 + #define GEN12_CTX_PRIORITY_HIGH FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 2) 95 + #define GEN12_CTX_PRIORITY_NORMAL FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 1) 96 + #define GEN12_CTX_PRIORITY_LOW FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 0) 97 + #define GEN8_CTX_ID_SHIFT 32 98 + #define GEN8_CTX_ID_WIDTH 21 99 + #define GEN11_SW_CTX_ID_SHIFT 37 100 + #define GEN11_SW_CTX_ID_WIDTH 11 101 + #define GEN11_ENGINE_CLASS_SHIFT 61 102 + #define GEN11_ENGINE_CLASS_WIDTH 3 103 + #define GEN11_ENGINE_INSTANCE_SHIFT 48 104 + #define GEN11_ENGINE_INSTANCE_WIDTH 6 105 + #define XEHP_SW_CTX_ID_SHIFT 39 106 + #define XEHP_SW_CTX_ID_WIDTH 16 107 + #define XEHP_SW_COUNTER_SHIFT 58 108 + #define XEHP_SW_COUNTER_WIDTH 6 109 + 110 + static inline u32 lrc_desc_priority(int prio) 111 + { 112 + if (prio > I915_PRIORITY_NORMAL) 113 + return GEN12_CTX_PRIORITY_HIGH; 114 + else if (prio < I915_PRIORITY_NORMAL) 115 + return GEN12_CTX_PRIORITY_LOW; 116 + else 117 + return GEN12_CTX_PRIORITY_NORMAL; 118 + } 74 119 75 120 #endif /* __INTEL_LRC_H__ */

+164 -32

drivers/gpu/drm/i915/gt/intel_migrate.c

··· 32 32 return true; 33 33 } 34 34 35 + static void xehpsdv_toggle_pdes(struct i915_address_space *vm, 36 + struct i915_page_table *pt, 37 + void *data) 38 + { 39 + struct insert_pte_data *d = data; 40 + 41 + /* 42 + * Insert a dummy PTE into every PT that will map to LMEM to ensure 43 + * we have a correctly setup PDE structure for later use. 44 + */ 45 + vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM); 46 + GEM_BUG_ON(!pt->is_compact); 47 + d->offset += SZ_2M; 48 + } 49 + 50 + static void xehpsdv_insert_pte(struct i915_address_space *vm, 51 + struct i915_page_table *pt, 52 + void *data) 53 + { 54 + struct insert_pte_data *d = data; 55 + 56 + /* 57 + * We are playing tricks here, since the actual pt, from the hw 58 + * pov, is only 256bytes with 32 entries, or 4096bytes with 512 59 + * entries, but we are still guaranteed that the physical 60 + * alignment is 64K underneath for the pt, and we are careful 61 + * not to access the space in the void. 62 + */ 63 + vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM); 64 + d->offset += SZ_64K; 65 + } 66 + 35 67 static void insert_pte(struct i915_address_space *vm, 36 68 struct i915_page_table *pt, 37 69 void *data) ··· 106 74 * i.e. within the same non-preemptible window so that we do not switch 107 75 * to another migration context that overwrites the PTE. 108 76 * 109 - * TODO: Add support for huge LMEM PTEs 77 + * This changes quite a bit on platforms with HAS_64K_PAGES support, 78 + * where we instead have three windows, each CHUNK_SIZE in size. The 79 + * first is reserved for mapping system-memory, and that just uses the 80 + * 512 entry layout using 4K GTT pages. The other two windows just map 81 + * lmem pages and must use the new compact 32 entry layout using 64K GTT 82 + * pages, which ensures we can address any lmem object that the user 83 + * throws at us. We then also use the xehpsdv_toggle_pdes as a way of 84 + * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the 85 + * compact layout for each of these page-tables, that fall within the 86 + * [CHUNK_SIZE, 3 * CHUNK_SIZE) range. 87 + * 88 + * We lay the ppGTT out as: 89 + * 90 + * [0, CHUNK_SZ) -> first window/object, maps smem 91 + * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src 92 + * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst 93 + * 94 + * For the PTE window it's also quite different, since each PTE must 95 + * point to some 64K page, one for each PT(since it's in lmem), and yet 96 + * each is only <= 4096bytes, but since the unused space within that PTE 97 + * range is never touched, this should be fine. 98 + * 99 + * So basically each PT now needs 64K of virtual memory, instead of 4K, 100 + * which looks like: 101 + * 102 + * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE 110 103 */ 111 104 112 105 vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY); ··· 142 85 err = -ENODEV; 143 86 goto err_vm; 144 87 } 88 + 89 + if (HAS_64K_PAGES(gt->i915)) 90 + stash.pt_sz = I915_GTT_PAGE_SIZE_64K; 145 91 146 92 /* 147 93 * Each engine instance is assigned its own chunk in the VM, so ··· 165 105 * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need 166 106 * 4x2 page directories for source/destination. 167 107 */ 168 - sz = 2 * CHUNK_SZ; 108 + if (HAS_64K_PAGES(gt->i915)) 109 + sz = 3 * CHUNK_SZ; 110 + else 111 + sz = 2 * CHUNK_SZ; 169 112 d.offset = base + sz; 170 113 171 114 /* 172 115 * We need another page directory setup so that we can write 173 116 * the 8x512 PTE in each chunk. 174 117 */ 175 - sz += (sz >> 12) * sizeof(u64); 118 + if (HAS_64K_PAGES(gt->i915)) 119 + sz += (sz / SZ_2M) * SZ_64K; 120 + else 121 + sz += (sz >> 12) * sizeof(u64); 176 122 177 123 err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz); 178 124 if (err) ··· 199 133 goto err_vm; 200 134 201 135 /* Now allow the GPU to rewrite the PTE via its own ppGTT */ 202 - vm->vm.foreach(&vm->vm, base, d.offset - base, insert_pte, &d); 136 + if (HAS_64K_PAGES(gt->i915)) { 137 + vm->vm.foreach(&vm->vm, base, d.offset - base, 138 + xehpsdv_insert_pte, &d); 139 + d.offset = base + CHUNK_SZ; 140 + vm->vm.foreach(&vm->vm, 141 + d.offset, 142 + 2 * CHUNK_SZ, 143 + xehpsdv_toggle_pdes, &d); 144 + } else { 145 + vm->vm.foreach(&vm->vm, base, d.offset - base, 146 + insert_pte, &d); 147 + } 203 148 } 204 149 205 150 return &vm->vm; ··· 346 269 u64 offset, 347 270 int length) 348 271 { 272 + bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915); 349 273 const u64 encode = rq->context->vm->pte_encode(0, cache_level, 350 274 is_lmem ? PTE_LM : 0); 351 275 struct intel_ring *ring = rq->ring; 352 - int total = 0; 276 + int pkt, dword_length; 277 + u32 total = 0; 278 + u32 page_size; 353 279 u32 *hdr, *cs; 354 - int pkt; 355 280 356 281 GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8); 357 282 283 + page_size = I915_GTT_PAGE_SIZE; 284 + dword_length = 0x400; 285 + 358 286 /* Compute the page directory offset for the target address range */ 359 - offset >>= 12; 360 - offset *= sizeof(u64); 361 - offset += 2 * CHUNK_SZ; 287 + if (has_64K_pages) { 288 + GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M)); 289 + 290 + offset /= SZ_2M; 291 + offset *= SZ_64K; 292 + offset += 3 * CHUNK_SZ; 293 + 294 + if (is_lmem) { 295 + page_size = I915_GTT_PAGE_SIZE_64K; 296 + dword_length = 0x40; 297 + } 298 + } else { 299 + offset >>= 12; 300 + offset *= sizeof(u64); 301 + offset += 2 * CHUNK_SZ; 302 + } 303 + 362 304 offset += (u64)rq->engine->instance << 32; 363 305 364 306 cs = intel_ring_begin(rq, 6); ··· 385 289 return PTR_ERR(cs); 386 290 387 291 /* Pack as many PTE updates as possible into a single MI command */ 388 - pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); 292 + pkt = min_t(int, dword_length, ring->space / sizeof(u32) + 5); 389 293 pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); 390 294 391 295 hdr = cs; ··· 395 299 396 300 do { 397 301 if (cs - hdr >= pkt) { 302 + int dword_rem; 303 + 398 304 *hdr += cs - hdr - 2; 399 305 *cs++ = MI_NOOP; 400 306 ··· 408 310 if (IS_ERR(cs)) 409 311 return PTR_ERR(cs); 410 312 411 - pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); 313 + dword_rem = dword_length; 314 + if (has_64K_pages) { 315 + if (IS_ALIGNED(total, SZ_2M)) { 316 + offset = round_up(offset, SZ_64K); 317 + } else { 318 + dword_rem = SZ_2M - (total & (SZ_2M - 1)); 319 + dword_rem /= page_size; 320 + dword_rem *= 2; 321 + } 322 + } 323 + 324 + pkt = min_t(int, dword_rem, ring->space / sizeof(u32) + 5); 412 325 pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); 413 326 414 327 hdr = cs; ··· 428 319 *cs++ = upper_32_bits(offset); 429 320 } 430 321 322 + GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size)); 323 + 431 324 *cs++ = lower_32_bits(encode | it->dma); 432 325 *cs++ = upper_32_bits(encode | it->dma); 433 326 434 327 offset += 8; 435 - total += I915_GTT_PAGE_SIZE; 328 + total += page_size; 436 329 437 - it->dma += I915_GTT_PAGE_SIZE; 330 + it->dma += page_size; 438 331 if (it->dma >= it->max) { 439 332 it->sg = __sg_next(it->sg); 440 333 if (!it->sg || sg_dma_len(it->sg) == 0) ··· 467 356 return height % 4 == 3 && height <= 8; 468 357 } 469 358 470 - static int emit_copy(struct i915_request *rq, int size) 359 + static int emit_copy(struct i915_request *rq, 360 + u32 dst_offset, u32 src_offset, int size) 471 361 { 472 362 const int ver = GRAPHICS_VER(rq->engine->i915); 473 363 u32 instance = rq->engine->instance; ··· 483 371 *cs++ = BLT_DEPTH_32 | PAGE_SIZE; 484 372 *cs++ = 0; 485 373 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 486 - *cs++ = CHUNK_SZ; /* dst offset */ 374 + *cs++ = dst_offset; 487 375 *cs++ = instance; 488 376 *cs++ = 0; 489 377 *cs++ = PAGE_SIZE; 490 - *cs++ = 0; /* src offset */ 378 + *cs++ = src_offset; 491 379 *cs++ = instance; 492 380 } else if (ver >= 8) { 493 381 *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); 494 382 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; 495 383 *cs++ = 0; 496 384 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 497 - *cs++ = CHUNK_SZ; /* dst offset */ 385 + *cs++ = dst_offset; 498 386 *cs++ = instance; 499 387 *cs++ = 0; 500 388 *cs++ = PAGE_SIZE; 501 - *cs++ = 0; /* src offset */ 389 + *cs++ = src_offset; 502 390 *cs++ = instance; 503 391 } else { 504 392 GEM_BUG_ON(instance); 505 393 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 506 394 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; 507 395 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; 508 - *cs++ = CHUNK_SZ; /* dst offset */ 396 + *cs++ = dst_offset; 509 397 *cs++ = PAGE_SIZE; 510 - *cs++ = 0; /* src offset */ 398 + *cs++ = src_offset; 511 399 } 512 400 513 401 intel_ring_advance(rq, cs); ··· 535 423 GEM_BUG_ON(ce->ring->size < SZ_64K); 536 424 537 425 do { 426 + u32 src_offset, dst_offset; 538 427 int len; 539 428 540 429 rq = i915_request_create(ce); ··· 563 450 if (err) 564 451 goto out_rq; 565 452 566 - len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, 0, 567 - CHUNK_SZ); 453 + src_offset = 0; 454 + dst_offset = CHUNK_SZ; 455 + if (HAS_64K_PAGES(ce->engine->i915)) { 456 + GEM_BUG_ON(!src_is_lmem && !dst_is_lmem); 457 + 458 + src_offset = 0; 459 + dst_offset = 0; 460 + if (src_is_lmem) 461 + src_offset = CHUNK_SZ; 462 + if (dst_is_lmem) 463 + dst_offset = 2 * CHUNK_SZ; 464 + } 465 + 466 + len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, 467 + src_offset, CHUNK_SZ); 568 468 if (len <= 0) { 569 469 err = len; 570 470 goto out_rq; 571 471 } 572 472 573 473 err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem, 574 - CHUNK_SZ, len); 474 + dst_offset, len); 575 475 if (err < 0) 576 476 goto out_rq; 577 477 if (err < len) { ··· 596 470 if (err) 597 471 goto out_rq; 598 472 599 - err = emit_copy(rq, len); 473 + err = emit_copy(rq, dst_offset, src_offset, len); 600 474 601 475 /* Arbitration is re-enabled between requests. */ 602 476 out_rq: ··· 614 488 return err; 615 489 } 616 490 617 - static int emit_clear(struct i915_request *rq, int size, u32 value) 491 + static int emit_clear(struct i915_request *rq, u64 offset, int size, u32 value) 618 492 { 619 493 const int ver = GRAPHICS_VER(rq->engine->i915); 620 - u32 instance = rq->engine->instance; 621 494 u32 *cs; 622 495 623 496 GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); 497 + 498 + offset += (u64)rq->engine->instance << 32; 624 499 625 500 cs = intel_ring_begin(rq, ver >= 8 ? 8 : 6); 626 501 if (IS_ERR(cs)) ··· 632 505 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; 633 506 *cs++ = 0; 634 507 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 635 - *cs++ = 0; /* offset */ 636 - *cs++ = instance; 508 + *cs++ = lower_32_bits(offset); 509 + *cs++ = upper_32_bits(offset); 637 510 *cs++ = value; 638 511 *cs++ = MI_NOOP; 639 512 } else { 640 - GEM_BUG_ON(instance); 513 + GEM_BUG_ON(upper_32_bits(offset)); 641 514 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 642 515 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; 643 516 *cs++ = 0; 644 517 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; 645 - *cs++ = 0; 518 + *cs++ = lower_32_bits(offset); 646 519 *cs++ = value; 647 520 } 648 521 ··· 669 542 GEM_BUG_ON(ce->ring->size < SZ_64K); 670 543 671 544 do { 545 + u32 offset; 672 546 int len; 673 547 674 548 rq = i915_request_create(ce); ··· 697 569 if (err) 698 570 goto out_rq; 699 571 700 - len = emit_pte(rq, &it, cache_level, is_lmem, 0, CHUNK_SZ); 572 + offset = 0; 573 + if (HAS_64K_PAGES(ce->engine->i915) && is_lmem) 574 + offset = CHUNK_SZ; 575 + 576 + len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ); 701 577 if (len <= 0) { 702 578 err = len; 703 579 goto out_rq; ··· 711 579 if (err) 712 580 goto out_rq; 713 581 714 - err = emit_clear(rq, len, value); 582 + err = emit_clear(rq, offset, len, value); 715 583 716 584 /* Arbitration is re-enabled between requests. */ 717 585 out_rq:

+13 -4

drivers/gpu/drm/i915/gt/intel_ppgtt.c

··· 12 12 #include "gen6_ppgtt.h" 13 13 #include "gen8_ppgtt.h" 14 14 15 - struct i915_page_table *alloc_pt(struct i915_address_space *vm) 15 + struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz) 16 16 { 17 17 struct i915_page_table *pt; 18 18 ··· 20 20 if (unlikely(!pt)) 21 21 return ERR_PTR(-ENOMEM); 22 22 23 - pt->base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); 23 + pt->base = vm->alloc_pt_dma(vm, sz); 24 24 if (IS_ERR(pt->base)) { 25 25 kfree(pt); 26 26 return ERR_PTR(-ENOMEM); 27 27 } 28 28 29 + pt->is_compact = false; 29 30 atomic_set(&pt->used, 0); 30 31 return pt; 31 32 } ··· 221 220 u64 size) 222 221 { 223 222 unsigned long count; 224 - int shift, n; 223 + int shift, n, pt_sz; 225 224 226 225 shift = vm->pd_shift; 227 226 if (!shift) 228 227 return 0; 229 228 229 + pt_sz = stash->pt_sz; 230 + if (!pt_sz) 231 + pt_sz = I915_GTT_PAGE_SIZE_4K; 232 + else 233 + GEM_BUG_ON(!IS_DGFX(vm->i915)); 234 + 235 + GEM_BUG_ON(!is_power_of_2(pt_sz)); 236 + 230 237 count = pd_count(size, shift); 231 238 while (count--) { 232 239 struct i915_page_table *pt; 233 240 234 - pt = alloc_pt(vm); 241 + pt = alloc_pt(vm, pt_sz); 235 242 if (IS_ERR(pt)) { 236 243 i915_vm_free_pt_stash(vm, stash); 237 244 return PTR_ERR(pt);

+28 -112

drivers/gpu/drm/i915/gt/intel_region_lmem.c

··· 14 14 #include "gt/intel_gt.h" 15 15 #include "gt/intel_gt_regs.h" 16 16 17 - static int init_fake_lmem_bar(struct intel_memory_region *mem) 18 - { 19 - struct drm_i915_private *i915 = mem->i915; 20 - struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 21 - unsigned long n; 22 - int ret; 23 - 24 - /* We want to 1:1 map the mappable aperture to our reserved region */ 25 - 26 - mem->fake_mappable.start = 0; 27 - mem->fake_mappable.size = resource_size(&mem->region); 28 - mem->fake_mappable.color = I915_COLOR_UNEVICTABLE; 29 - 30 - ret = drm_mm_reserve_node(&ggtt->vm.mm, &mem->fake_mappable); 31 - if (ret) 32 - return ret; 33 - 34 - mem->remap_addr = dma_map_resource(i915->drm.dev, 35 - mem->region.start, 36 - mem->fake_mappable.size, 37 - DMA_BIDIRECTIONAL, 38 - DMA_ATTR_FORCE_CONTIGUOUS); 39 - if (dma_mapping_error(i915->drm.dev, mem->remap_addr)) { 40 - drm_mm_remove_node(&mem->fake_mappable); 41 - return -EINVAL; 42 - } 43 - 44 - for (n = 0; n < mem->fake_mappable.size >> PAGE_SHIFT; ++n) { 45 - ggtt->vm.insert_page(&ggtt->vm, 46 - mem->remap_addr + (n << PAGE_SHIFT), 47 - n << PAGE_SHIFT, 48 - I915_CACHE_NONE, 0); 49 - } 50 - 51 - mem->region = (struct resource)DEFINE_RES_MEM(mem->remap_addr, 52 - mem->fake_mappable.size); 53 - 54 - return 0; 55 - } 56 - 57 - static void release_fake_lmem_bar(struct intel_memory_region *mem) 58 - { 59 - if (!drm_mm_node_allocated(&mem->fake_mappable)) 60 - return; 61 - 62 - drm_mm_remove_node(&mem->fake_mappable); 63 - 64 - dma_unmap_resource(mem->i915->drm.dev, 65 - mem->remap_addr, 66 - mem->fake_mappable.size, 67 - DMA_BIDIRECTIONAL, 68 - DMA_ATTR_FORCE_CONTIGUOUS); 69 - } 70 - 71 17 static int 72 18 region_lmem_release(struct intel_memory_region *mem) 73 19 { ··· 21 75 22 76 ret = intel_region_ttm_fini(mem); 23 77 io_mapping_fini(&mem->iomap); 24 - release_fake_lmem_bar(mem); 25 78 26 79 return ret; 27 80 } ··· 30 85 { 31 86 int ret; 32 87 33 - if (mem->i915->params.fake_lmem_start) { 34 - ret = init_fake_lmem_bar(mem); 35 - GEM_BUG_ON(ret); 36 - } 37 - 38 88 if (!io_mapping_init_wc(&mem->iomap, 39 89 mem->io_start, 40 - resource_size(&mem->region))) { 41 - ret = -EIO; 42 - goto out_no_io; 43 - } 90 + mem->io_size)) 91 + return -EIO; 44 92 45 93 ret = intel_region_ttm_init(mem); 46 94 if (ret) ··· 43 105 44 106 out_no_buddy: 45 107 io_mapping_fini(&mem->iomap); 46 - out_no_io: 47 - release_fake_lmem_bar(mem); 48 108 49 109 return ret; 50 110 } ··· 52 116 .release = region_lmem_release, 53 117 .init_object = __i915_gem_ttm_object_init, 54 118 }; 55 - 56 - struct intel_memory_region * 57 - intel_gt_setup_fake_lmem(struct intel_gt *gt) 58 - { 59 - struct drm_i915_private *i915 = gt->i915; 60 - struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 61 - struct intel_memory_region *mem; 62 - resource_size_t mappable_end; 63 - resource_size_t io_start; 64 - resource_size_t start; 65 - 66 - if (!HAS_LMEM(i915)) 67 - return ERR_PTR(-ENODEV); 68 - 69 - if (!i915->params.fake_lmem_start) 70 - return ERR_PTR(-ENODEV); 71 - 72 - GEM_BUG_ON(i915_ggtt_has_aperture(to_gt(i915)->ggtt)); 73 - 74 - /* Your mappable aperture belongs to me now! */ 75 - mappable_end = pci_resource_len(pdev, 2); 76 - io_start = pci_resource_start(pdev, 2); 77 - start = i915->params.fake_lmem_start; 78 - 79 - mem = intel_memory_region_create(i915, 80 - start, 81 - mappable_end, 82 - PAGE_SIZE, 83 - io_start, 84 - INTEL_MEMORY_LOCAL, 85 - 0, 86 - &intel_region_lmem_ops); 87 - if (!IS_ERR(mem)) { 88 - drm_info(&i915->drm, "Intel graphics fake LMEM: %pR\n", 89 - &mem->region); 90 - drm_info(&i915->drm, 91 - "Intel graphics fake LMEM IO start: %llx\n", 92 - (u64)mem->io_start); 93 - drm_info(&i915->drm, "Intel graphics fake LMEM size: %llx\n", 94 - (u64)resource_size(&mem->region)); 95 - } 96 - 97 - return mem; 98 - } 99 119 100 120 static bool get_legacy_lowmem_region(struct intel_uncore *uncore, 101 121 u64 *start, u32 *size) ··· 99 207 if (!IS_DGFX(i915)) 100 208 return ERR_PTR(-ENODEV); 101 209 102 - /* Stolen starts from GSMBASE on DG1 */ 103 - lmem_size = intel_uncore_read64(uncore, GEN12_GSMBASE); 210 + if (HAS_FLAT_CCS(i915)) { 211 + u64 tile_stolen, flat_ccs_base; 212 + 213 + lmem_size = pci_resource_len(pdev, 2); 214 + flat_ccs_base = intel_gt_read_register(gt, XEHPSDV_FLAT_CCS_BASE_ADDR); 215 + flat_ccs_base = (flat_ccs_base >> XEHPSDV_CCS_BASE_SHIFT) * SZ_64K; 216 + 217 + if (GEM_WARN_ON(lmem_size < flat_ccs_base)) 218 + return ERR_PTR(-ENODEV); 219 + 220 + tile_stolen = lmem_size - flat_ccs_base; 221 + 222 + /* If the FLAT_CCS_BASE_ADDR register is not populated, flag an error */ 223 + if (tile_stolen == lmem_size) 224 + drm_err(&i915->drm, 225 + "CCS_BASE_ADDR register did not have expected value\n"); 226 + 227 + lmem_size -= tile_stolen; 228 + } else { 229 + /* Stolen starts from GSMBASE without CCS */ 230 + lmem_size = intel_uncore_read64(&i915->uncore, GEN12_GSMBASE); 231 + } 232 + 104 233 105 234 io_start = pci_resource_start(pdev, 2); 106 235 if (GEM_WARN_ON(lmem_size > pci_resource_len(pdev, 2))) ··· 134 221 lmem_size, 135 222 min_page_size, 136 223 io_start, 224 + lmem_size, 137 225 INTEL_MEMORY_LOCAL, 138 226 0, 139 227 &intel_region_lmem_ops); ··· 148 234 drm_dbg(&i915->drm, "Local memory: %pR\n", &mem->region); 149 235 drm_dbg(&i915->drm, "Local memory IO start: %pa\n", 150 236 &mem->io_start); 237 + drm_info(&i915->drm, "Local memory IO size: %pa\n", 238 + &mem->io_size); 151 239 drm_info(&i915->drm, "Local memory available: %pa\n", 152 240 &lmem_size); 153 241

-3

drivers/gpu/drm/i915/gt/intel_region_lmem.h

··· 10 10 11 11 struct intel_memory_region *intel_gt_setup_lmem(struct intel_gt *gt); 12 12 13 - struct intel_memory_region * 14 - intel_gt_setup_fake_lmem(struct intel_gt *gt); 15 - 16 13 #endif /* !__INTEL_REGION_LMEM_H */

+1 -1

drivers/gpu/drm/i915/gt/intel_rps.c

··· 1486 1486 1487 1487 if (has_busy_stats(rps)) 1488 1488 intel_rps_set_timer(rps); 1489 - else if (GRAPHICS_VER(i915) >= 6) 1489 + else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1490 1490 intel_rps_set_interrupts(rps); 1491 1491 else 1492 1492 /* Ironlake currently uses intel_ips.ko */ {}

+14 -3

drivers/gpu/drm/i915/gt/intel_sseu.c

··· 32 32 return total; 33 33 } 34 34 35 - u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) 35 + static u32 36 + _intel_sseu_get_subslices(const struct sseu_dev_info *sseu, 37 + const u8 *subslice_mask, u8 slice) 36 38 { 37 39 int i, offset = slice * sseu->ss_stride; 38 40 u32 mask = 0; ··· 42 40 GEM_BUG_ON(slice >= sseu->max_slices); 43 41 44 42 for (i = 0; i < sseu->ss_stride; i++) 45 - mask |= (u32)sseu->subslice_mask[offset + i] << 46 - i * BITS_PER_BYTE; 43 + mask |= (u32)subslice_mask[offset + i] << i * BITS_PER_BYTE; 47 44 48 45 return mask; 46 + } 47 + 48 + u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) 49 + { 50 + return _intel_sseu_get_subslices(sseu, sseu->subslice_mask, slice); 51 + } 52 + 53 + u32 intel_sseu_get_compute_subslices(const struct sseu_dev_info *sseu) 54 + { 55 + return _intel_sseu_get_subslices(sseu, sseu->compute_subslice_mask, 0); 49 56 } 50 57 51 58 void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice,

+3 -1

drivers/gpu/drm/i915/gt/intel_sseu.h

··· 103 103 unsigned int 104 104 intel_sseu_subslices_per_slice(const struct sseu_dev_info *sseu, u8 slice); 105 105 106 - u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice); 106 + u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice); 107 + 108 + u32 intel_sseu_get_compute_subslices(const struct sseu_dev_info *sseu); 107 109 108 110 void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice, 109 111 u8 *subslice_mask, u32 ss_mask);

+99 -39

drivers/gpu/drm/i915/gt/intel_workarounds.c

··· 684 684 wa_masked_en(wal, SLICE_COMMON_ECO_CHICKEN1, 685 685 MSC_MSAA_REODER_BUF_BYPASS_DISABLE); 686 686 687 - /* Wa_22012532006:dg2 */ 688 - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) || 689 - IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) 690 - wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, 691 - DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA); 687 + /* Wa_14014947963:dg2 */ 688 + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) || 689 + IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915)) 690 + wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000); 692 691 } 693 692 694 693 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, ··· 1343 1344 /* Wa_1409757795:xehpsdv */ 1344 1345 wa_write_or(wal, SCCGCTL94DC, CG3DDISURB); 1345 1346 1346 - /* Wa_18011725039:xehpsdv */ 1347 - if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) { 1348 - wa_masked_dis(wal, MLTICTXCTL, TDONRENDER); 1349 - wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH); 1350 - } 1351 - 1352 1347 /* Wa_16011155590:xehpsdv */ 1353 1348 if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) 1354 1349 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, ··· 1379 1386 GAMTLBVEBOX0_CLKGATE_DIS); 1380 1387 } 1381 1388 1382 - /* Wa_14012362059:xehpsdv */ 1383 - wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); 1384 - 1385 1389 /* Wa_16012725990:xehpsdv */ 1386 1390 if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER)) 1387 1391 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS); 1388 1392 1389 1393 /* Wa_14011060649:xehpsdv */ 1390 1394 wa_14011060649(gt, wal); 1391 - 1392 - /* Wa_14014368820:xehpsdv */ 1393 - wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | 1394 - GLOBAL_INVALIDATION_MODE); 1395 1395 } 1396 1396 1397 1397 static void ··· 1426 1440 } 1427 1441 1428 1442 if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)) { 1429 - /* Wa_14010680813:dg2_g10 */ 1430 - wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS | 1431 - EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS); 1432 - 1433 1443 /* Wa_14010948348:dg2_g10 */ 1434 1444 wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS); 1435 1445 ··· 1472 1490 wa_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS); 1473 1491 } 1474 1492 1475 - if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0) || 1476 - IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0)) { 1477 - /* Wa_14012362059:dg2 */ 1478 - wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); 1479 - } 1480 - 1481 - /* Wa_1509235366:dg2 */ 1482 - wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | 1483 - GLOBAL_INVALIDATION_MODE); 1484 - 1485 1493 /* Wa_14014830051:dg2 */ 1486 1494 wa_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN); 1487 1495 ··· 1480 1508 * recommended tuning settings documented in the bspec's 1481 1509 * performance guide section. 1482 1510 */ 1483 - wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); 1484 1511 wa_write_or(wal, GEN12_SQCM, EN_32B_ACCESS); 1485 - 1486 - /* Wa_18018781329:dg2 */ 1487 - wa_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); 1488 - wa_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); 1489 - wa_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB); 1490 - wa_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB); 1491 1512 } 1492 1513 1493 1514 static void ··· 1908 1943 RING_FORCE_TO_NONPRIV_RANGE_4); 1909 1944 1910 1945 break; 1946 + case COMPUTE_CLASS: 1947 + /* Wa_16011157294:dg2_g10 */ 1948 + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) 1949 + whitelist_reg(w, GEN9_CTX_PREEMPT_REG); 1950 + break; 1911 1951 default: 1912 1952 break; 1913 1953 } ··· 2019 2049 if (IS_DG2(i915)) { 2020 2050 /* Wa_14015227452:dg2 */ 2021 2051 wa_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE); 2052 + 2053 + /* Wa_1509235366:dg2 */ 2054 + wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | 2055 + GLOBAL_INVALIDATION_MODE); 2056 + 2057 + /* 2058 + * The following are not actually "workarounds" but rather 2059 + * recommended tuning settings documented in the bspec's 2060 + * performance guide section. 2061 + */ 2062 + wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); 2063 + 2064 + /* Wa_18018781329:dg2 */ 2065 + wa_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); 2066 + wa_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); 2067 + wa_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB); 2068 + wa_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB); 2022 2069 } 2023 2070 2024 2071 if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) { ··· 2135 2148 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) || 2136 2149 IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) 2137 2150 wa_write_or(wal, RT_CTRL, DIS_NULL_QUERY); 2151 + 2152 + /* Wa_22012532006:dg2 */ 2153 + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) || 2154 + IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) 2155 + wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, 2156 + DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA); 2157 + 2158 + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) { 2159 + /* Wa_14010680813:dg2_g10 */ 2160 + wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS | 2161 + EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS); 2162 + } 2163 + 2164 + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0) || 2165 + IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) { 2166 + /* Wa_14012362059:dg2 */ 2167 + wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); 2168 + } 2138 2169 2139 2170 if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) || 2140 2171 IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) { ··· 2573 2568 } 2574 2569 } 2575 2570 2571 + /* 2572 + * The workarounds in this function apply to shared registers in 2573 + * the general render reset domain that aren't tied to a 2574 + * specific engine. Since all render+compute engines get reset 2575 + * together, and the contents of these registers are lost during 2576 + * the shared render domain reset, we'll define such workarounds 2577 + * here and then add them to just a single RCS or CCS engine's 2578 + * workaround list (whichever engine has the XXXX flag). 2579 + */ 2580 + static void 2581 + general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2582 + { 2583 + struct drm_i915_private *i915 = engine->i915; 2584 + 2585 + if (IS_XEHPSDV(i915)) { 2586 + /* Wa_1409954639 */ 2587 + wa_masked_en(wal, 2588 + GEN8_ROW_CHICKEN, 2589 + SYSTOLIC_DOP_CLOCK_GATING_DIS); 2590 + 2591 + /* Wa_1607196519 */ 2592 + wa_masked_en(wal, 2593 + GEN9_ROW_CHICKEN4, 2594 + GEN12_DISABLE_GRF_CLEAR); 2595 + 2596 + /* Wa_14010670810:xehpsdv */ 2597 + wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE); 2598 + 2599 + /* Wa_14010449647:xehpsdv */ 2600 + wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, 2601 + GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); 2602 + 2603 + /* Wa_18011725039:xehpsdv */ 2604 + if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) { 2605 + wa_masked_dis(wal, MLTICTXCTL, TDONRENDER); 2606 + wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH); 2607 + } 2608 + 2609 + /* Wa_14012362059:xehpsdv */ 2610 + wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); 2611 + 2612 + /* Wa_14014368820:xehpsdv */ 2613 + wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | 2614 + GLOBAL_INVALIDATION_MODE); 2615 + } 2616 + } 2617 + 2576 2618 static void 2577 2619 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2578 2620 { ··· 2627 2575 return; 2628 2576 2629 2577 engine_fake_wa_init(engine, wal); 2578 + 2579 + /* 2580 + * These are common workarounds that just need to applied 2581 + * to a single RCS/CCS engine's workaround list since 2582 + * they're reset as part of the general render domain reset. 2583 + */ 2584 + if (engine->class == RENDER_CLASS) 2585 + general_render_compute_wa_init(engine, wal); 2630 2586 2631 2587 if (engine->class == RENDER_CLASS) 2632 2588 rcs_engine_wa_init(engine, wal);

+32

drivers/gpu/drm/i915/gt/shmem_utils.c

··· 3 3 * Copyright © 2020 Intel Corporation 4 4 */ 5 5 6 + #include <linux/iosys-map.h> 6 7 #include <linux/mm.h> 7 8 #include <linux/pagemap.h> 8 9 #include <linux/shmem_fs.h> ··· 118 117 119 118 len -= this; 120 119 ptr += this; 120 + off = 0; 121 + } 122 + 123 + return 0; 124 + } 125 + 126 + int shmem_read_to_iosys_map(struct file *file, loff_t off, 127 + struct iosys_map *map, size_t map_off, size_t len) 128 + { 129 + unsigned long pfn; 130 + 131 + for (pfn = off >> PAGE_SHIFT; len; pfn++) { 132 + unsigned int this = 133 + min_t(size_t, PAGE_SIZE - offset_in_page(off), len); 134 + struct page *page; 135 + void *vaddr; 136 + 137 + page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, 138 + GFP_KERNEL); 139 + if (IS_ERR(page)) 140 + return PTR_ERR(page); 141 + 142 + vaddr = kmap(page); 143 + iosys_map_memcpy_to(map, map_off, vaddr + offset_in_page(off), 144 + this); 145 + mark_page_accessed(page); 146 + kunmap(page); 147 + put_page(page); 148 + 149 + len -= this; 150 + map_off += this; 121 151 off = 0; 122 152 } 123 153

+3

drivers/gpu/drm/i915/gt/shmem_utils.h

··· 8 8 9 9 #include <linux/types.h> 10 10 11 + struct iosys_map; 11 12 struct drm_i915_gem_object; 12 13 struct file; 13 14 ··· 18 17 void *shmem_pin_map(struct file *file); 19 18 void shmem_unpin_map(struct file *file, void *ptr); 20 19 20 + int shmem_read_to_iosys_map(struct file *file, loff_t off, 21 + struct iosys_map *map, size_t map_off, size_t len); 21 22 int shmem_read(struct file *file, loff_t off, void *dst, size_t len); 22 23 int shmem_write(struct file *file, loff_t off, void *src, size_t len); 23 24

+4 -3

drivers/gpu/drm/i915/gt/uc/intel_guc.h

··· 6 6 #ifndef _INTEL_GUC_H_ 7 7 #define _INTEL_GUC_H_ 8 8 9 - #include <linux/xarray.h> 10 9 #include <linux/delay.h> 10 + #include <linux/iosys-map.h> 11 + #include <linux/xarray.h> 11 12 12 13 #include "intel_uncore.h" 13 14 #include "intel_guc_fw.h" ··· 147 146 148 147 /** @ads_vma: object allocated to hold the GuC ADS */ 149 148 struct i915_vma *ads_vma; 150 - /** @ads_blob: contents of the GuC ADS */ 151 - struct __guc_ads_blob *ads_blob; 149 + /** @ads_map: contents of the GuC ADS */ 150 + struct iosys_map ads_map; 152 151 /** @ads_regset_size: size of the save/restore regsets in the ADS */ 153 152 u32 ads_regset_size; 154 153 /**

+138 -100

drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c

··· 60 60 struct guc_mmio_reg regset[0]; 61 61 } __packed; 62 62 63 + #define ads_blob_read(guc_, field_) \ 64 + iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) 65 + 66 + #define ads_blob_write(guc_, field_, val_) \ 67 + iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ 68 + field_, val_) 69 + 70 + #define info_map_write(map_, field_, val_) \ 71 + iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) 72 + 73 + #define info_map_read(map_, field_) \ 74 + iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) 75 + 63 76 static u32 guc_ads_regset_size(struct intel_guc *guc) 64 77 { 65 78 GEM_BUG_ON(!guc->ads_regset_size); ··· 136 123 guc_ads_private_data_size(guc); 137 124 } 138 125 139 - static void guc_policies_init(struct intel_guc *guc, struct guc_policies *policies) 126 + static void guc_policies_init(struct intel_guc *guc) 140 127 { 141 128 struct intel_gt *gt = guc_to_gt(guc); 142 129 struct drm_i915_private *i915 = gt->i915; 130 + u32 global_flags = 0; 143 131 144 - policies->dpc_promote_time = GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US; 145 - policies->max_num_work_items = GLOBAL_POLICY_MAX_NUM_WI; 132 + ads_blob_write(guc, policies.dpc_promote_time, 133 + GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 134 + ads_blob_write(guc, policies.max_num_work_items, 135 + GLOBAL_POLICY_MAX_NUM_WI); 146 136 147 - policies->global_flags = 0; 148 137 if (i915->params.reset < 2) 149 - policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 138 + global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 150 139 151 - policies->is_valid = 1; 140 + ads_blob_write(guc, policies.global_flags, global_flags); 141 + ads_blob_write(guc, policies.is_valid, 1); 152 142 } 153 143 154 144 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 155 145 struct drm_printer *dp) 156 146 { 157 - struct __guc_ads_blob *blob = guc->ads_blob; 158 - 159 - if (unlikely(!blob)) 147 + if (unlikely(iosys_map_is_null(&guc->ads_map))) 160 148 return; 161 149 162 150 drm_printf(dp, "Global scheduling policies:\n"); 163 - drm_printf(dp, " DPC promote time = %u\n", blob->policies.dpc_promote_time); 164 - drm_printf(dp, " Max num work items = %u\n", blob->policies.max_num_work_items); 165 - drm_printf(dp, " Flags = %u\n", blob->policies.global_flags); 151 + drm_printf(dp, " DPC promote time = %u\n", 152 + ads_blob_read(guc, policies.dpc_promote_time)); 153 + drm_printf(dp, " Max num work items = %u\n", 154 + ads_blob_read(guc, policies.max_num_work_items)); 155 + drm_printf(dp, " Flags = %u\n", 156 + ads_blob_read(guc, policies.global_flags)); 166 157 } 167 158 168 159 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) ··· 181 164 182 165 int intel_guc_global_policies_update(struct intel_guc *guc) 183 166 { 184 - struct __guc_ads_blob *blob = guc->ads_blob; 185 167 struct intel_gt *gt = guc_to_gt(guc); 168 + u32 scheduler_policies; 186 169 intel_wakeref_t wakeref; 187 170 int ret; 188 171 189 - if (!blob) 172 + if (iosys_map_is_null(&guc->ads_map)) 190 173 return -EOPNOTSUPP; 191 174 192 - GEM_BUG_ON(!blob->ads.scheduler_policies); 175 + scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); 176 + GEM_BUG_ON(!scheduler_policies); 193 177 194 - guc_policies_init(guc, &blob->policies); 178 + guc_policies_init(guc); 195 179 196 180 if (!intel_guc_is_ready(guc)) 197 181 return 0; 198 182 199 183 with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref) 200 - ret = guc_action_policies_update(guc, blob->ads.scheduler_policies); 184 + ret = guc_action_policies_update(guc, scheduler_policies); 201 185 202 186 return ret; 203 187 } 204 188 205 189 static void guc_mapping_table_init(struct intel_gt *gt, 206 - struct guc_gt_system_info *system_info) 190 + struct iosys_map *info_map) 207 191 { 208 192 unsigned int i, j; 209 193 struct intel_engine_cs *engine; ··· 213 195 /* Table must be set to invalid values for entries not used */ 214 196 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 215 197 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 216 - system_info->mapping_table[i][j] = 217 - GUC_MAX_INSTANCES_PER_CLASS; 198 + info_map_write(info_map, mapping_table[i][j], 199 + GUC_MAX_INSTANCES_PER_CLASS); 218 200 219 201 for_each_engine(engine, gt, id) { 220 202 u8 guc_class = engine_class_to_guc_class(engine->class); 221 203 222 - system_info->mapping_table[guc_class][ilog2(engine->logical_mask)] = 223 - engine->instance; 204 + info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], 205 + engine->instance); 224 206 } 225 207 } 226 208 ··· 335 317 ret |= GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); 336 318 ret |= GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); 337 319 320 + if (engine->class == RENDER_CLASS && 321 + CCS_MASK(engine->gt)) 322 + ret |= GUC_MMIO_REG_ADD(regset, GEN12_RCU_MODE, true); 323 + 338 324 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 339 325 ret |= GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); 340 326 ··· 387 365 return ret; 388 366 } 389 367 390 - static void guc_mmio_reg_state_init(struct intel_guc *guc, 391 - struct __guc_ads_blob *blob) 368 + static void guc_mmio_reg_state_init(struct intel_guc *guc) 392 369 { 393 370 struct intel_gt *gt = guc_to_gt(guc); 394 371 struct intel_engine_cs *engine; 395 - struct guc_mmio_reg *ads_registers; 396 372 enum intel_engine_id id; 397 373 u32 addr_ggtt, offset; 398 374 399 375 offset = guc_ads_regset_offset(guc); 400 376 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 401 - ads_registers = (struct guc_mmio_reg *)(((u8 *)blob) + offset); 402 377 403 - memcpy(ads_registers, guc->ads_regset, guc->ads_regset_size); 378 + iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, 379 + guc->ads_regset_size); 404 380 405 381 for_each_engine(engine, gt, id) { 406 382 u32 count = guc->ads_regset_count[id]; 407 - struct guc_mmio_reg_set *ads_reg_set; 408 383 u8 guc_class; 409 384 410 385 /* Class index is checked in class converter */ 411 386 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 412 387 413 388 guc_class = engine_class_to_guc_class(engine->class); 414 - ads_reg_set = &blob->ads.reg_state_list[guc_class][engine->instance]; 415 389 416 390 if (!count) { 417 - ads_reg_set->address = 0; 418 - ads_reg_set->count = 0; 391 + ads_blob_write(guc, 392 + ads.reg_state_list[guc_class][engine->instance].address, 393 + 0); 394 + ads_blob_write(guc, 395 + ads.reg_state_list[guc_class][engine->instance].count, 396 + 0); 419 397 continue; 420 398 } 421 399 422 - ads_reg_set->address = addr_ggtt; 423 - ads_reg_set->count = count; 400 + ads_blob_write(guc, 401 + ads.reg_state_list[guc_class][engine->instance].address, 402 + addr_ggtt); 403 + ads_blob_write(guc, 404 + ads.reg_state_list[guc_class][engine->instance].count, 405 + count); 424 406 425 407 addr_ggtt += count * sizeof(struct guc_mmio_reg); 426 408 } 427 409 } 428 410 429 411 static void fill_engine_enable_masks(struct intel_gt *gt, 430 - struct guc_gt_system_info *info) 412 + struct iosys_map *info_map) 431 413 { 432 - info->engine_enabled_masks[GUC_RENDER_CLASS] = 1; 433 - info->engine_enabled_masks[GUC_BLITTER_CLASS] = 1; 434 - info->engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); 435 - info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); 414 + info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], 1); 415 + info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); 416 + info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 1); 417 + info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); 418 + info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); 436 419 } 437 420 438 421 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 439 422 #define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE) 440 - static int guc_prep_golden_context(struct intel_guc *guc, 441 - struct __guc_ads_blob *blob) 423 + static int guc_prep_golden_context(struct intel_guc *guc) 442 424 { 443 425 struct intel_gt *gt = guc_to_gt(guc); 444 426 u32 addr_ggtt, offset; 445 427 u32 total_size = 0, alloc_size, real_size; 446 428 u8 engine_class, guc_class; 447 - struct guc_gt_system_info *info, local_info; 429 + struct guc_gt_system_info local_info; 430 + struct iosys_map info_map; 448 431 449 432 /* 450 433 * Reserve the memory for the golden contexts and point GuC at it but ··· 463 436 * GuC will also validate that the LRC base + size fall within the 464 437 * allowed GGTT range. 465 438 */ 466 - if (blob) { 439 + if (!iosys_map_is_null(&guc->ads_map)) { 467 440 offset = guc_ads_golden_ctxt_offset(guc); 468 441 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 469 - info = &blob->system_info; 442 + info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 443 + offsetof(struct __guc_ads_blob, system_info)); 470 444 } else { 471 445 memset(&local_info, 0, sizeof(local_info)); 472 - info = &local_info; 473 - fill_engine_enable_masks(gt, info); 446 + iosys_map_set_vaddr(&info_map, &local_info); 447 + fill_engine_enable_masks(gt, &info_map); 474 448 } 475 449 476 450 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { ··· 480 452 481 453 guc_class = engine_class_to_guc_class(engine_class); 482 454 483 - if (!info->engine_enabled_masks[guc_class]) 455 + if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) 484 456 continue; 485 457 486 458 real_size = intel_engine_context_size(gt, engine_class); 487 459 alloc_size = PAGE_ALIGN(real_size); 488 460 total_size += alloc_size; 489 461 490 - if (!blob) 462 + if (iosys_map_is_null(&guc->ads_map)) 491 463 continue; 492 464 493 465 /* ··· 501 473 * what comes before it in the context image (which is identical 502 474 * on all engines). 503 475 */ 504 - blob->ads.eng_state_size[guc_class] = real_size - LRC_SKIP_SIZE; 505 - blob->ads.golden_context_lrca[guc_class] = addr_ggtt; 476 + ads_blob_write(guc, ads.eng_state_size[guc_class], 477 + real_size - LRC_SKIP_SIZE); 478 + ads_blob_write(guc, ads.golden_context_lrca[guc_class], 479 + addr_ggtt); 480 + 506 481 addr_ggtt += alloc_size; 507 482 } 508 483 509 - if (!blob) 510 - return total_size; 484 + /* Make sure current size matches what we calculated previously */ 485 + if (guc->ads_golden_ctxt_size) 486 + GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 511 487 512 - GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 513 488 return total_size; 514 489 } 515 490 ··· 536 505 537 506 static void guc_init_golden_context(struct intel_guc *guc) 538 507 { 539 - struct __guc_ads_blob *blob = guc->ads_blob; 540 508 struct intel_engine_cs *engine; 541 509 struct intel_gt *gt = guc_to_gt(guc); 542 - u32 addr_ggtt, offset; 543 - u32 total_size = 0, alloc_size, real_size; 510 + unsigned long offset; 511 + u32 addr_ggtt, total_size = 0, alloc_size, real_size; 544 512 u8 engine_class, guc_class; 545 - u8 *ptr; 546 513 547 514 if (!intel_uc_uses_guc_submission(&gt->uc)) 548 515 return; 549 516 550 - GEM_BUG_ON(!blob); 517 + GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 551 518 552 519 /* 553 520 * Go back and fill in the golden context data now that it is ··· 553 524 */ 554 525 offset = guc_ads_golden_ctxt_offset(guc); 555 526 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 556 - ptr = ((u8 *)blob) + offset; 557 527 558 528 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 559 529 if (engine_class == OTHER_CLASS) 560 530 continue; 561 531 562 532 guc_class = engine_class_to_guc_class(engine_class); 563 - 564 - if (!blob->system_info.engine_enabled_masks[guc_class]) 533 + if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) 565 534 continue; 566 535 567 536 real_size = intel_engine_context_size(gt, engine_class); ··· 570 543 if (!engine) { 571 544 drm_err(&gt->i915->drm, "No engine state recorded for class %d!\n", 572 545 engine_class); 573 - blob->ads.eng_state_size[guc_class] = 0; 574 - blob->ads.golden_context_lrca[guc_class] = 0; 546 + ads_blob_write(guc, ads.eng_state_size[guc_class], 0); 547 + ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); 575 548 continue; 576 549 } 577 550 578 - GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != 551 + GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != 579 552 real_size - LRC_SKIP_SIZE); 580 - GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); 553 + GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); 554 + 581 555 addr_ggtt += alloc_size; 582 556 583 - shmem_read(engine->default_state, 0, ptr, real_size); 584 - ptr += alloc_size; 557 + shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, 558 + offset, real_size); 559 + offset += alloc_size; 585 560 } 586 561 587 562 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 588 563 } 589 564 590 - static void guc_capture_list_init(struct intel_guc *guc, struct __guc_ads_blob *blob) 565 + static void guc_capture_list_init(struct intel_guc *guc) 591 566 { 592 567 int i, j; 593 568 u32 addr_ggtt, offset; ··· 601 572 602 573 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 603 574 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { 604 - blob->ads.capture_instance[i][j] = addr_ggtt; 605 - blob->ads.capture_class[i][j] = addr_ggtt; 575 + ads_blob_write(guc, ads.capture_instance[i][j], addr_ggtt); 576 + ads_blob_write(guc, ads.capture_class[i][j], addr_ggtt); 606 577 } 607 578 608 - blob->ads.capture_global[i] = addr_ggtt; 579 + ads_blob_write(guc, ads.capture_global[i], addr_ggtt); 609 580 } 610 581 } 611 582 ··· 613 584 { 614 585 struct intel_gt *gt = guc_to_gt(guc); 615 586 struct drm_i915_private *i915 = gt->i915; 616 - struct __guc_ads_blob *blob = guc->ads_blob; 587 + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 588 + offsetof(struct __guc_ads_blob, system_info)); 617 589 u32 base; 618 590 619 591 /* GuC scheduling policies */ 620 - guc_policies_init(guc, &blob->policies); 592 + guc_policies_init(guc); 621 593 622 594 /* System info */ 623 - fill_engine_enable_masks(gt, &blob->system_info); 595 + fill_engine_enable_masks(gt, &info_map); 624 596 625 - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = 626 - hweight8(gt->info.sseu.slice_mask); 627 - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = 628 - gt->info.vdbox_sfc_access; 597 + ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], 598 + hweight8(gt->info.sseu.slice_mask)); 599 + ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], 600 + gt->info.vdbox_sfc_access); 629 601 630 602 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 631 603 u32 distdbreg = intel_uncore_read(gt->uncore, 632 604 GEN12_DIST_DBS_POPULATED); 633 - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = 634 - ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & 635 - GEN12_DOORBELLS_PER_SQIDI) + 1; 605 + ads_blob_write(guc, 606 + system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 607 + ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) 608 + & GEN12_DOORBELLS_PER_SQIDI) + 1); 636 609 } 637 610 638 611 /* Golden contexts for re-initialising after a watchdog reset */ 639 - guc_prep_golden_context(guc, blob); 612 + guc_prep_golden_context(guc); 640 613 641 - guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); 614 + guc_mapping_table_init(guc_to_gt(guc), &info_map); 642 615 643 616 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 644 617 645 618 /* Capture list for hang debug */ 646 - guc_capture_list_init(guc, blob); 619 + guc_capture_list_init(guc); 647 620 648 621 /* ADS */ 649 - blob->ads.scheduler_policies = base + ptr_offset(blob, policies); 650 - blob->ads.gt_system_info = base + ptr_offset(blob, system_info); 622 + ads_blob_write(guc, ads.scheduler_policies, base + 623 + offsetof(struct __guc_ads_blob, policies)); 624 + ads_blob_write(guc, ads.gt_system_info, base + 625 + offsetof(struct __guc_ads_blob, system_info)); 651 626 652 627 /* MMIO save/restore list */ 653 - guc_mmio_reg_state_init(guc, blob); 628 + guc_mmio_reg_state_init(guc); 654 629 655 630 /* Private Data */ 656 - blob->ads.private_data = base + guc_ads_private_data_offset(guc); 631 + ads_blob_write(guc, ads.private_data, base + 632 + guc_ads_private_data_offset(guc)); 657 633 658 634 i915_gem_object_flush_map(guc->ads_vma->obj); 659 635 } ··· 672 638 */ 673 639 int intel_guc_ads_create(struct intel_guc *guc) 674 640 { 641 + void *ads_blob; 675 642 u32 size; 676 643 int ret; 677 644 ··· 688 653 guc->ads_regset_size = ret; 689 654 690 655 /* Likewise the golden contexts: */ 691 - ret = guc_prep_golden_context(guc, NULL); 656 + ret = guc_prep_golden_context(guc); 692 657 if (ret < 0) 693 658 return ret; 694 659 guc->ads_golden_ctxt_size = ret; ··· 697 662 size = guc_ads_blob_size(guc); 698 663 699 664 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 700 - (void **)&guc->ads_blob); 665 + &ads_blob); 701 666 if (ret) 702 667 return ret; 668 + 669 + if (i915_gem_object_is_lmem(guc->ads_vma->obj)) 670 + iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); 671 + else 672 + iosys_map_set_vaddr(&guc->ads_map, ads_blob); 703 673 704 674 __guc_ads_init(guc); 705 675 ··· 726 686 void intel_guc_ads_destroy(struct intel_guc *guc) 727 687 { 728 688 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 729 - guc->ads_blob = NULL; 689 + iosys_map_clear(&guc->ads_map); 730 690 kfree(guc->ads_regset); 731 691 } 732 692 ··· 738 698 if (!size) 739 699 return; 740 700 741 - memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, 742 - size); 701 + iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), 702 + 0, size); 743 703 } 744 704 745 705 /** ··· 762 722 763 723 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 764 724 { 765 - struct __guc_ads_blob *blob = guc->ads_blob; 766 - u32 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 767 - u32 offset = base + ptr_offset(blob, engine_usage); 768 - 769 - return offset; 725 + return intel_guc_ggtt_offset(guc, guc->ads_vma) + 726 + offsetof(struct __guc_ads_blob, engine_usage); 770 727 } 771 728 772 - struct guc_engine_usage_record *intel_guc_engine_usage(struct intel_engine_cs *engine) 729 + struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) 773 730 { 774 731 struct intel_guc *guc = &engine->gt->uc.guc; 775 - struct __guc_ads_blob *blob = guc->ads_blob; 776 732 u8 guc_class = engine_class_to_guc_class(engine->class); 733 + size_t offset = offsetof(struct __guc_ads_blob, 734 + engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); 777 735 778 - return &blob->engine_usage.engines[guc_class][ilog2(engine->logical_mask)]; 736 + return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); 779 737 }

+2 -1

drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h

··· 7 7 #define _INTEL_GUC_ADS_H_ 8 8 9 9 #include <linux/types.h> 10 + #include <linux/iosys-map.h> 10 11 11 12 struct intel_guc; 12 13 struct drm_printer; ··· 19 18 void intel_guc_ads_reset(struct intel_guc *guc); 20 19 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 21 20 struct drm_printer *p); 22 - struct guc_engine_usage_record *intel_guc_engine_usage(struct intel_engine_cs *engine); 21 + struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine); 23 22 u32 intel_guc_engine_usage_offset(struct intel_guc *guc); 24 23 25 24 #endif

+23 -9

drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h

··· 46 46 #define GUC_VIDEO_CLASS 1 47 47 #define GUC_VIDEOENHANCE_CLASS 2 48 48 #define GUC_BLITTER_CLASS 3 49 - #define GUC_RESERVED_CLASS 4 50 - #define GUC_LAST_ENGINE_CLASS GUC_RESERVED_CLASS 49 + #define GUC_COMPUTE_CLASS 4 50 + #define GUC_LAST_ENGINE_CLASS GUC_COMPUTE_CLASS 51 51 #define GUC_MAX_ENGINE_CLASSES 16 52 52 #define GUC_MAX_INSTANCES_PER_CLASS 32 53 53 ··· 156 156 FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, c) \ 157 157 ) 158 158 159 + /* the GuC arrays don't include OTHER_CLASS */ 160 + static u8 engine_class_guc_class_map[] = { 161 + [RENDER_CLASS] = GUC_RENDER_CLASS, 162 + [COPY_ENGINE_CLASS] = GUC_BLITTER_CLASS, 163 + [VIDEO_DECODE_CLASS] = GUC_VIDEO_CLASS, 164 + [VIDEO_ENHANCEMENT_CLASS] = GUC_VIDEOENHANCE_CLASS, 165 + [COMPUTE_CLASS] = GUC_COMPUTE_CLASS, 166 + }; 167 + 168 + static u8 guc_class_engine_class_map[] = { 169 + [GUC_RENDER_CLASS] = RENDER_CLASS, 170 + [GUC_BLITTER_CLASS] = COPY_ENGINE_CLASS, 171 + [GUC_VIDEO_CLASS] = VIDEO_DECODE_CLASS, 172 + [GUC_VIDEOENHANCE_CLASS] = VIDEO_ENHANCEMENT_CLASS, 173 + [GUC_COMPUTE_CLASS] = COMPUTE_CLASS, 174 + }; 175 + 159 176 static inline u8 engine_class_to_guc_class(u8 class) 160 177 { 161 - BUILD_BUG_ON(GUC_RENDER_CLASS != RENDER_CLASS); 162 - BUILD_BUG_ON(GUC_BLITTER_CLASS != COPY_ENGINE_CLASS); 163 - BUILD_BUG_ON(GUC_VIDEO_CLASS != VIDEO_DECODE_CLASS); 164 - BUILD_BUG_ON(GUC_VIDEOENHANCE_CLASS != VIDEO_ENHANCEMENT_CLASS); 178 + BUILD_BUG_ON(ARRAY_SIZE(engine_class_guc_class_map) != MAX_ENGINE_CLASS + 1); 165 179 GEM_BUG_ON(class > MAX_ENGINE_CLASS || class == OTHER_CLASS); 166 180 167 - return class; 181 + return engine_class_guc_class_map[class]; 168 182 } 169 183 170 184 static inline u8 guc_class_to_engine_class(u8 guc_class) 171 185 { 186 + BUILD_BUG_ON(ARRAY_SIZE(guc_class_engine_class_map) != GUC_LAST_ENGINE_CLASS + 1); 172 187 GEM_BUG_ON(guc_class > GUC_LAST_ENGINE_CLASS); 173 - GEM_BUG_ON(guc_class == GUC_RESERVED_CLASS); 174 188 175 - return guc_class; 189 + return guc_class_engine_class_map[guc_class]; 176 190 } 177 191 178 192 /* Work item for submitting workloads into work queue of GuC. */

+4 -3

drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c

··· 11 11 #include "intel_mchbar_regs.h" 12 12 #include "gt/intel_gt.h" 13 13 #include "gt/intel_gt_regs.h" 14 + #include "gt/intel_rps.h" 14 15 15 16 static inline struct intel_guc *slpc_to_guc(struct intel_guc_slpc *slpc) 16 17 { ··· 116 115 { 117 116 u32 request[] = { 118 117 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, 119 - SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 2), 118 + SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1), 120 119 id, 121 120 }; 122 121 ··· 580 579 581 580 static void slpc_get_rp_values(struct intel_guc_slpc *slpc) 582 581 { 582 + struct intel_rps *rps = &slpc_to_gt(slpc)->rps; 583 583 u32 rp_state_cap; 584 584 585 - rp_state_cap = intel_uncore_read(slpc_to_gt(slpc)->uncore, 586 - GEN6_RP_STATE_CAP); 585 + rp_state_cap = intel_rps_read_state_cap(rps); 587 586 588 587 slpc->rp0_freq = REG_FIELD_GET(RP0_CAP_MASK, rp_state_cap) * 589 588 GT_FREQUENCY_MULTIPLIER;

+33 -30

drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c

··· 174 174 __maybe_unused 175 175 static bool sched_state_is_init(struct intel_context *ce) 176 176 { 177 - /* 178 - * XXX: Kernel contexts can have SCHED_STATE_NO_LOCK_REGISTERED after 179 - * suspend. 180 - */ 181 - return !(ce->guc_state.sched_state &= 177 + /* Kernel contexts can have SCHED_STATE_REGISTERED after suspend. */ 178 + return !(ce->guc_state.sched_state & 182 179 ~(SCHED_STATE_BLOCKED_MASK | SCHED_STATE_REGISTERED)); 183 180 } 184 181 ··· 1136 1139 *prev_start = ((u64)gt_stamp_hi << 32) | new_start; 1137 1140 } 1138 1141 1142 + #define record_read(map_, field_) \ 1143 + iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_) 1144 + 1139 1145 /* 1140 1146 * GuC updates shared memory and KMD reads it. Since this is not synchronized, 1141 1147 * we run into a race where the value read is inconsistent. Sometimes the ··· 1153 1153 static void __get_engine_usage_record(struct intel_engine_cs *engine, 1154 1154 u32 *last_in, u32 *id, u32 *total) 1155 1155 { 1156 - struct guc_engine_usage_record *rec = intel_guc_engine_usage(engine); 1156 + struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine); 1157 1157 int i = 0; 1158 1158 1159 1159 do { 1160 - *last_in = READ_ONCE(rec->last_switch_in_stamp); 1161 - *id = READ_ONCE(rec->current_context_index); 1162 - *total = READ_ONCE(rec->total_runtime); 1160 + *last_in = record_read(&rec_map, last_switch_in_stamp); 1161 + *id = record_read(&rec_map, current_context_index); 1162 + *total = record_read(&rec_map, total_runtime); 1163 1163 1164 - if (READ_ONCE(rec->last_switch_in_stamp) == *last_in && 1165 - READ_ONCE(rec->current_context_index) == *id && 1166 - READ_ONCE(rec->total_runtime) == *total) 1164 + if (record_read(&rec_map, last_switch_in_stamp) == *last_in && 1165 + record_read(&rec_map, current_context_index) == *id && 1166 + record_read(&rec_map, total_runtime) == *total) 1167 1167 break; 1168 1168 } while (++i < 6); 1169 1169 } ··· 1818 1818 */ 1819 1819 GEM_BUG_ON(!guc->lrc_desc_pool); 1820 1820 1821 - xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); 1822 - 1823 - spin_lock_init(&guc->submission_state.lock); 1824 - INIT_LIST_HEAD(&guc->submission_state.guc_id_list); 1825 - ida_init(&guc->submission_state.guc_ids); 1826 - INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts); 1827 - INIT_WORK(&guc->submission_state.destroyed_worker, 1828 - destroyed_worker_func); 1829 - INIT_WORK(&guc->submission_state.reset_fail_worker, 1830 - reset_fail_worker_func); 1831 - 1832 1821 guc->submission_state.guc_ids_bitmap = 1833 1822 bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL); 1834 1823 if (!guc->submission_state.guc_ids_bitmap) 1835 1824 return -ENOMEM; 1836 1825 1837 - spin_lock_init(&guc->timestamp.lock); 1838 - INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping); 1839 1826 guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ; 1840 1827 guc->timestamp.shift = gpm_timestamp_shift(gt); 1841 1828 ··· 3595 3608 setup_hwsp(engine); 3596 3609 start_engine(engine); 3597 3610 3611 + if (engine->class == RENDER_CLASS) 3612 + xehp_enable_ccs_engines(engine); 3613 + 3598 3614 return 0; 3599 3615 } 3600 3616 ··· 3779 3789 guc_default_irqs(engine); 3780 3790 guc_init_breadcrumbs(engine); 3781 3791 3782 - if (engine->class == RENDER_CLASS) 3792 + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) 3783 3793 rcs_submission_override(engine); 3784 3794 3785 3795 lrc_init_wa_ctx(engine); ··· 3821 3831 3822 3832 void intel_guc_submission_init_early(struct intel_guc *guc) 3823 3833 { 3834 + xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); 3835 + 3836 + spin_lock_init(&guc->submission_state.lock); 3837 + INIT_LIST_HEAD(&guc->submission_state.guc_id_list); 3838 + ida_init(&guc->submission_state.guc_ids); 3839 + INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts); 3840 + INIT_WORK(&guc->submission_state.destroyed_worker, 3841 + destroyed_worker_func); 3842 + INIT_WORK(&guc->submission_state.reset_fail_worker, 3843 + reset_fail_worker_func); 3844 + 3845 + spin_lock_init(&guc->timestamp.lock); 3846 + INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping); 3847 + 3824 3848 guc->submission_state.num_guc_ids = GUC_MAX_LRC_DESCRIPTORS; 3825 3849 guc->submission_supported = __guc_submission_supported(guc); 3826 3850 guc->submission_selected = __guc_submission_selected(guc); ··· 4026 4022 capture_error_state(guc, ce); 4027 4023 guc_context_replay(ce); 4028 4024 } else { 4029 - drm_err(&guc_to_gt(guc)->i915->drm, 4030 - "Invalid GuC engine reset notificaion for 0x%04X on %s: banned = %d, blocked = %d", 4031 - ce->guc_id.id, ce->engine->name, intel_context_is_banned(ce), 4032 - context_blocked(ce)); 4025 + drm_info(&guc_to_gt(guc)->i915->drm, 4026 + "Ignoring context reset notification of banned context 0x%04X on %s", 4027 + ce->guc_id.id, ce->engine->name); 4033 4028 } 4034 4029 } 4035 4030

+4

drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c

··· 154 154 int ret; 155 155 156 156 for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { 157 + /* We don't support breadcrumb handshake on these classes */ 158 + if (class == COMPUTE_CLASS || class == RENDER_CLASS) 159 + continue; 160 + 157 161 ret = __intel_guc_multi_lrc_basic(gt, class); 158 162 if (ret) 159 163 return ret;

-15

drivers/gpu/drm/i915/i915_driver.c

··· 835 835 if (!i915->params.nuclear_pageflip && match_info->graphics.ver < 5) 836 836 i915->drm.driver_features &= ~DRIVER_ATOMIC; 837 837 838 - /* 839 - * Check if we support fake LMEM -- for now we only unleash this for 840 - * the live selftests(test-and-exit). 841 - */ 842 - #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 843 - if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) { 844 - if (GRAPHICS_VER(i915) >= 9 && i915_selftest.live < 0 && 845 - i915->params.fake_lmem_start) { 846 - mkwrite_device_info(i915)->memory_regions = 847 - REGION_SMEM | REGION_LMEM | REGION_STOLEN_SMEM; 848 - GEM_BUG_ON(!HAS_LMEM(i915)); 849 - } 850 - } 851 - #endif 852 - 853 838 ret = pci_enable_device(pdev); 854 839 if (ret) 855 840 goto out_fini;

+20 -10

drivers/gpu/drm/i915/i915_drv.h

··· 1147 1147 #define IS_ICL_WITH_PORT_F(dev_priv) \ 1148 1148 IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF) 1149 1149 1150 - #define IS_TGL_U(dev_priv) \ 1151 - IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULT) 1152 - 1153 - #define IS_TGL_Y(dev_priv) \ 1154 - IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULX) 1150 + #define IS_TGL_UY(dev_priv) \ 1151 + IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_UY) 1155 1152 1156 1153 #define IS_SKL_GRAPHICS_STEP(p, since, until) (IS_SKYLAKE(p) && IS_GRAPHICS_STEP(p, since, until)) 1157 1154 ··· 1167 1170 IS_DISPLAY_STEP(__i915, since, until)) 1168 1171 1169 1172 #define IS_TGL_UY_GRAPHICS_STEP(__i915, since, until) \ 1170 - ((IS_TGL_U(__i915) || IS_TGL_Y(__i915)) && \ 1173 + (IS_TGL_UY(__i915) && \ 1171 1174 IS_GRAPHICS_STEP(__i915, since, until)) 1172 1175 1173 1176 #define IS_TGL_GRAPHICS_STEP(__i915, since, until) \ 1174 - (IS_TIGERLAKE(__i915) && !(IS_TGL_U(__i915) || IS_TGL_Y(__i915)) && \ 1177 + (IS_TIGERLAKE(__i915) && !IS_TGL_UY(__i915)) && \ 1175 1178 IS_GRAPHICS_STEP(__i915, since, until)) 1176 1179 1177 1180 #define IS_RKL_DISPLAY_STEP(p, since, until) \ ··· 1241 1244 ENGINE_INSTANCES_MASK(gt, VCS0, I915_MAX_VCS) 1242 1245 #define VEBOX_MASK(gt) \ 1243 1246 ENGINE_INSTANCES_MASK(gt, VECS0, I915_MAX_VECS) 1247 + #define CCS_MASK(gt) \ 1248 + ENGINE_INSTANCES_MASK(gt, CCS0, I915_MAX_CCS) 1244 1249 1245 1250 /* 1246 1251 * The Gen7 cmdparser copies the scanned buffer to the ggtt for execution ··· 1339 1340 1340 1341 /* 1341 1342 * Set this flag, when platform requires 64K GTT page sizes or larger for 1342 - * device local memory access. Also this flag implies that we require or 1343 - * at least support the compact PT layout for the ppGTT when using the 64K 1344 - * GTT pages. 1343 + * device local memory access. 1345 1344 */ 1346 1345 #define HAS_64K_PAGES(dev_priv) (INTEL_INFO(dev_priv)->has_64k_pages) 1346 + 1347 + /* 1348 + * Set this flag when platform doesn't allow both 64k pages and 4k pages in 1349 + * the same PT. this flag means we need to support compact PT layout for the 1350 + * ppGTT when using the 64K GTT pages. 1351 + */ 1352 + #define NEEDS_COMPACT_PT(dev_priv) (INTEL_INFO(dev_priv)->needs_compact_pt) 1347 1353 1348 1354 #define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) 1349 1355 1350 1356 #define HAS_REGION(i915, i) (INTEL_INFO(i915)->memory_regions & (i)) 1351 1357 #define HAS_LMEM(i915) HAS_REGION(i915, REGION_LMEM) 1358 + 1359 + /* 1360 + * Platform has the dedicated compression control state for each lmem surfaces 1361 + * stored in lmem to support the 3D and media compression formats. 1362 + */ 1363 + #define HAS_FLAT_CCS(dev_priv) (INTEL_INFO(dev_priv)->has_flat_ccs) 1352 1364 1353 1365 #define HAS_GT_UC(dev_priv) (INTEL_INFO(dev_priv)->has_gt_uc) 1354 1366

-5

drivers/gpu/drm/i915/i915_params.c

··· 195 195 "Enable support for Intel GVT-g graphics virtualization host support(default:false)"); 196 196 #endif 197 197 198 - #if IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) 199 - i915_param_named_unsafe(fake_lmem_start, ulong, 0400, 200 - "Fake LMEM start offset (default: 0)"); 201 - #endif 202 - 203 198 #if CONFIG_DRM_I915_REQUEST_TIMEOUT 204 199 i915_param_named_unsafe(request_timeout_ms, uint, 0600, 205 200 "Default request/fence/batch buffer expiration timeout.");

-1

drivers/gpu/drm/i915/i915_params.h

··· 72 72 param(int, fastboot, -1, 0600) \ 73 73 param(int, enable_dpcd_backlight, -1, 0600) \ 74 74 param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \ 75 - param(unsigned long, fake_lmem_start, 0, IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) ? 0400 : 0) \ 76 75 param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \ 77 76 /* leave bools at the end to not create holes */ \ 78 77 param(bool, enable_hangcheck, true, 0600) \

+3

drivers/gpu/drm/i915/i915_pci.c

··· 1005 1005 XE_HP_PAGE_SIZES, \ 1006 1006 .dma_mask_size = 46, \ 1007 1007 .has_64bit_reloc = 1, \ 1008 + .has_flat_ccs = 1, \ 1008 1009 .has_global_mocs = 1, \ 1009 1010 .has_gt_uc = 1, \ 1010 1011 .has_llc = 1, \ ··· 1031 1030 PLATFORM(INTEL_XEHPSDV), 1032 1031 .display = { }, 1033 1032 .has_64k_pages = 1, 1033 + .needs_compact_pt = 1, 1034 1034 .platform_engine_mask = 1035 1035 BIT(RCS0) | BIT(BCS0) | 1036 1036 BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) | ··· 1051 1049 PLATFORM(INTEL_DG2), 1052 1050 .has_guc_deprivilege = 1, 1053 1051 .has_64k_pages = 1, 1052 + .needs_compact_pt = 1, 1054 1053 .platform_engine_mask = 1055 1054 BIT(RCS0) | BIT(BCS0) | 1056 1055 BIT(VECS0) | BIT(VECS1) |

+4

drivers/gpu/drm/i915/i915_perf.c

··· 4374 4374 4375 4375 /* XXX const struct i915_perf_ops! */ 4376 4376 4377 + /* i915_perf is not enabled for DG2 yet */ 4378 + if (IS_DG2(i915)) 4379 + return; 4380 + 4377 4381 perf->oa_formats = oa_formats; 4378 4382 if (IS_HASWELL(i915)) { 4379 4383 perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr;

+18 -6

drivers/gpu/drm/i915/i915_reg.h

··· 971 971 #define GEN11_VEBOX2_RING_BASE 0x1d8000 972 972 #define XEHP_VEBOX3_RING_BASE 0x1e8000 973 973 #define XEHP_VEBOX4_RING_BASE 0x1f8000 974 + #define GEN12_COMPUTE0_RING_BASE 0x1a000 975 + #define GEN12_COMPUTE1_RING_BASE 0x1c000 976 + #define GEN12_COMPUTE2_RING_BASE 0x1e000 977 + #define GEN12_COMPUTE3_RING_BASE 0x26000 974 978 #define BLT_RING_BASE 0x22000 975 979 976 980 ··· 8464 8460 #define SGGI_DIS REG_BIT(15) 8465 8461 #define SGR_DIS REG_BIT(13) 8466 8462 8463 + #define XEHPSDV_FLAT_CCS_BASE_ADDR _MMIO(0x4910) 8464 + #define XEHPSDV_CCS_BASE_SHIFT 8 8465 + 8466 + /* gamt regs */ 8467 + #define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) 8468 + #define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW 0x67F1427F /* max/min for LRA1/2 */ 8469 + #define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV 0x5FF101FF /* max/min for LRA1/2 */ 8470 + #define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL 0x67F1427F /* " " */ 8471 + #define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT 0x5FF101FF /* " " */ 8472 + 8473 + #define MMCD_MISC_CTRL _MMIO(0x4ddc) /* skl+ */ 8474 + #define MMCD_PCLA (1 << 31) 8475 + #define MMCD_HOTSPOT_EN (1 << 27) 8476 + 8467 8477 #define _ICL_PHY_MISC_A 0x64C00 8468 8478 #define _ICL_PHY_MISC_B 0x64C04 8469 8479 #define _DG2_PHY_MISC_TC1 0x64C14 /* TC1="PHY E" but offset as if "PHY F" */ ··· 8840 8822 #define DSB_CTRL(pipe, id) _MMIO(DSBSL_INSTANCE(pipe, id) + 0x8) 8841 8823 #define DSB_ENABLE (1 << 31) 8842 8824 #define DSB_STATUS (1 << 0) 8843 - 8844 - #define TGL_ROOT_DEVICE_ID 0x9A00 8845 - #define TGL_ROOT_DEVICE_MASK 0xFF00 8846 - #define TGL_ROOT_DEVICE_SKU_MASK 0xF 8847 - #define TGL_ROOT_DEVICE_SKU_ULX 0x2 8848 - #define TGL_ROOT_DEVICE_SKU_ULT 0x4 8849 8825 8850 8826 #define CLKREQ_POLICY _MMIO(0x101038) 8851 8827 #define CLKREQ_POLICY_MEM_UP_OVRD REG_BIT(1)

+85 -2

drivers/gpu/drm/i915/i915_ttm_buddy_manager.c

··· 19 19 struct drm_buddy mm; 20 20 struct list_head reserved; 21 21 struct mutex lock; 22 + unsigned long visible_size; 23 + unsigned long visible_avail; 24 + unsigned long visible_reserved; 22 25 u64 default_page_size; 23 26 }; 24 27 ··· 82 79 lpfn = pages; 83 80 } 84 81 85 - if (size > mm->size) { 82 + if (size > lpfn << PAGE_SHIFT) { 86 83 err = -E2BIG; 87 84 goto err_free_res; 88 85 } ··· 90 87 n_pages = size >> ilog2(mm->chunk_size); 91 88 92 89 mutex_lock(&bman->lock); 90 + if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) { 91 + mutex_unlock(&bman->lock); 92 + err = -ENOSPC; 93 + goto err_free_res; 94 + } 95 + 93 96 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 94 97 (u64)lpfn << PAGE_SHIFT, 95 98 (u64)n_pages << PAGE_SHIFT, ··· 115 106 &bman_res->blocks); 116 107 mutex_unlock(&bman->lock); 117 108 } 109 + 110 + if (lpfn <= bman->visible_size) { 111 + bman_res->used_visible_size = bman_res->base.num_pages; 112 + } else { 113 + struct drm_buddy_block *block; 114 + 115 + list_for_each_entry(block, &bman_res->blocks, link) { 116 + unsigned long start = 117 + drm_buddy_block_offset(block) >> PAGE_SHIFT; 118 + 119 + if (start < bman->visible_size) { 120 + unsigned long end = start + 121 + (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 122 + 123 + bman_res->used_visible_size += 124 + min(end, bman->visible_size) - start; 125 + } 126 + } 127 + } 128 + 129 + if (bman_res->used_visible_size) { 130 + mutex_lock(&bman->lock); 131 + bman->visible_avail -= bman_res->used_visible_size; 132 + mutex_unlock(&bman->lock); 133 + } 134 + 135 + if (place->lpfn - place->fpfn == n_pages) 136 + bman_res->base.start = place->fpfn; 137 + else if (lpfn <= bman->visible_size) 138 + bman_res->base.start = 0; 139 + else 140 + bman_res->base.start = bman->visible_size; 118 141 119 142 *res = &bman_res->base; 120 143 return 0; ··· 169 128 170 129 mutex_lock(&bman->lock); 171 130 drm_buddy_free_list(&bman->mm, &bman_res->blocks); 131 + bman->visible_avail += bman_res->used_visible_size; 172 132 mutex_unlock(&bman->lock); 173 133 174 134 ttm_resource_fini(man, res); ··· 185 143 mutex_lock(&bman->lock); 186 144 drm_printf(printer, "default_page_size: %lluKiB\n", 187 145 bman->default_page_size >> 10); 146 + drm_printf(printer, "visible_avail: %lluMiB\n", 147 + (u64)bman->visible_avail << PAGE_SHIFT >> 20); 148 + drm_printf(printer, "visible_size: %lluMiB\n", 149 + (u64)bman->visible_size << PAGE_SHIFT >> 20); 150 + drm_printf(printer, "visible_reserved: %lluMiB\n", 151 + (u64)bman->visible_reserved << PAGE_SHIFT >> 20); 188 152 189 153 drm_buddy_print(&bman->mm, printer); 190 154 ··· 212 164 * @type: Memory type we want to manage 213 165 * @use_tt: Set use_tt for the manager 214 166 * @size: The size in bytes to manage 167 + * @visible_size: The CPU visible size in bytes to manage 215 168 * @default_page_size: The default minimum page size in bytes for allocations, 216 169 * this must be at least as large as @chunk_size, and can be overridden by 217 170 * setting the BO page_alignment, to be larger or smaller as needed. ··· 236 187 */ 237 188 int i915_ttm_buddy_man_init(struct ttm_device *bdev, 238 189 unsigned int type, bool use_tt, 239 - u64 size, u64 default_page_size, 190 + u64 size, u64 visible_size, u64 default_page_size, 240 191 u64 chunk_size) 241 192 { 242 193 struct ttm_resource_manager *man; ··· 255 206 INIT_LIST_HEAD(&bman->reserved); 256 207 GEM_BUG_ON(default_page_size < chunk_size); 257 208 bman->default_page_size = default_page_size; 209 + bman->visible_size = visible_size >> PAGE_SHIFT; 210 + bman->visible_avail = bman->visible_size; 258 211 259 212 man = &bman->manager; 260 213 man->use_tt = use_tt; ··· 301 250 mutex_lock(&bman->lock); 302 251 drm_buddy_free_list(mm, &bman->reserved); 303 252 drm_buddy_fini(mm); 253 + bman->visible_avail += bman->visible_reserved; 254 + WARN_ON_ONCE(bman->visible_avail != bman->visible_size); 304 255 mutex_unlock(&bman->lock); 305 256 306 257 ttm_resource_manager_cleanup(man); ··· 326 273 { 327 274 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 328 275 struct drm_buddy *mm = &bman->mm; 276 + unsigned long fpfn = start >> PAGE_SHIFT; 329 277 unsigned long flags = 0; 330 278 int ret; 331 279 ··· 338 284 size, mm->chunk_size, 339 285 &bman->reserved, 340 286 flags); 287 + 288 + if (fpfn < bman->visible_size) { 289 + unsigned long lpfn = fpfn + (size >> PAGE_SHIFT); 290 + unsigned long visible = min(lpfn, bman->visible_size) - fpfn; 291 + 292 + bman->visible_reserved += visible; 293 + bman->visible_avail -= visible; 294 + } 341 295 mutex_unlock(&bman->lock); 342 296 343 297 return ret; 344 298 } 345 299 300 + /** 301 + * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion 302 + * in pages. 303 + * @man: The buddy allocator ttm manager 304 + */ 305 + u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man) 306 + { 307 + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 308 + 309 + return bman->visible_size; 310 + } 311 + 312 + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 313 + void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, 314 + u64 size) 315 + { 316 + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 317 + 318 + bman->visible_size = size; 319 + } 320 + #endif

+12 -1

drivers/gpu/drm/i915/i915_ttm_buddy_manager.h

··· 21 21 * @base: struct ttm_resource base class we extend 22 22 * @blocks: the list of struct i915_buddy_block for this resource/allocation 23 23 * @flags: DRM_BUDDY_*_ALLOCATION flags 24 + * @used_visible_size: How much of this resource, if any, uses the CPU visible 25 + * portion, in pages. 24 26 * @mm: the struct i915_buddy_mm for this resource 25 27 * 26 28 * Extends the struct ttm_resource to manage an address space allocation with ··· 32 30 struct ttm_resource base; 33 31 struct list_head blocks; 34 32 unsigned long flags; 33 + unsigned long used_visible_size; 35 34 struct drm_buddy *mm; 36 35 }; 37 36 ··· 51 48 52 49 int i915_ttm_buddy_man_init(struct ttm_device *bdev, 53 50 unsigned type, bool use_tt, 54 - u64 size, u64 default_page_size, u64 chunk_size); 51 + u64 size, u64 visible_size, 52 + u64 default_page_size, u64 chunk_size); 55 53 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, 56 54 unsigned int type); 57 55 58 56 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, 59 57 u64 start, u64 size); 58 + 59 + u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man); 60 + 61 + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 62 + void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, 63 + u64 size); 64 + #endif 60 65 61 66 #endif

+74 -12

drivers/gpu/drm/i915/i915_vma.c

··· 540 540 void __iomem *ptr; 541 541 int err; 542 542 543 + if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY)) 544 + return IO_ERR_PTR(-EINVAL); 545 + 543 546 if (!i915_gem_object_is_lmem(vma->obj)) { 544 547 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { 545 548 err = -ENODEV; ··· 760 757 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); 761 758 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 762 759 760 + alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj)); 761 + /* 762 + * for compact-pt we round up the reservation to prevent 763 + * any smaller pages being used within the same PDE 764 + */ 765 + if (NEEDS_COMPACT_PT(vma->vm->i915)) 766 + size = round_up(size, alignment); 767 + 763 768 /* If binding the object/GGTT view requires more space than the entire 764 769 * aperture has, reject it early before evicting everything in a vain 765 770 * attempt to find space. ··· 780 769 } 781 770 782 771 color = 0; 772 + 783 773 if (i915_vm_has_cache_coloring(vma->vm)) 784 774 color = vma->obj->cache_level; 785 775 ··· 1621 1609 void i915_vma_release(struct kref *ref) 1622 1610 { 1623 1611 struct i915_vma *vma = container_of(ref, typeof(*vma), ref); 1612 + 1613 + i915_vm_put(vma->vm); 1614 + i915_active_fini(&vma->active); 1615 + GEM_WARN_ON(vma->resource); 1616 + i915_vma_free(vma); 1617 + } 1618 + 1619 + static void force_unbind(struct i915_vma *vma) 1620 + { 1621 + if (!drm_mm_node_allocated(&vma->node)) 1622 + return; 1623 + 1624 + atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 1625 + WARN_ON(__i915_vma_unbind(vma)); 1626 + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1627 + } 1628 + 1629 + static void release_references(struct i915_vma *vma) 1630 + { 1624 1631 struct drm_i915_gem_object *obj = vma->obj; 1625 1632 1626 - if (drm_mm_node_allocated(&vma->node)) { 1627 - mutex_lock(&vma->vm->mutex); 1628 - atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 1629 - WARN_ON(__i915_vma_unbind(vma)); 1630 - mutex_unlock(&vma->vm->mutex); 1631 - GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1632 - } 1633 1633 GEM_BUG_ON(i915_vma_is_active(vma)); 1634 1634 1635 1635 spin_lock(&obj->vma.lock); ··· 1651 1627 spin_unlock(&obj->vma.lock); 1652 1628 1653 1629 __i915_vma_remove_closed(vma); 1654 - i915_vm_put(vma->vm); 1655 1630 1656 - i915_active_fini(&vma->active); 1657 - GEM_WARN_ON(vma->resource); 1658 - i915_vma_free(vma); 1631 + __i915_vma_put(vma); 1632 + } 1633 + 1634 + /** 1635 + * i915_vma_destroy_locked - Remove all weak reference to the vma and put 1636 + * the initial reference. 1637 + * 1638 + * This function should be called when it's decided the vma isn't needed 1639 + * anymore. The caller must assure that it doesn't race with another lookup 1640 + * plus destroy, typically by taking an appropriate reference. 1641 + * 1642 + * Current callsites are 1643 + * - __i915_gem_object_pages_fini() 1644 + * - __i915_vm_close() - Blocks the above function by taking a reference on 1645 + * the object. 1646 + * - __i915_vma_parked() - Blocks the above functions by taking an open-count on 1647 + * the vm and a reference on the object. 1648 + * 1649 + * Because of locks taken during destruction, a vma is also guaranteed to 1650 + * stay alive while the following locks are held if it was looked up while 1651 + * holding one of the locks: 1652 + * - vm->mutex 1653 + * - obj->vma.lock 1654 + * - gt->closed_lock 1655 + * 1656 + * A vma user can also temporarily keep the vma alive while holding a vma 1657 + * reference. 1658 + */ 1659 + void i915_vma_destroy_locked(struct i915_vma *vma) 1660 + { 1661 + lockdep_assert_held(&vma->vm->mutex); 1662 + 1663 + force_unbind(vma); 1664 + release_references(vma); 1665 + } 1666 + 1667 + void i915_vma_destroy(struct i915_vma *vma) 1668 + { 1669 + mutex_lock(&vma->vm->mutex); 1670 + force_unbind(vma); 1671 + mutex_unlock(&vma->vm->mutex); 1672 + release_references(vma); 1659 1673 } 1660 1674 1661 1675 void i915_vma_parked(struct intel_gt *gt) ··· 1727 1665 1728 1666 if (i915_gem_object_trylock(obj, NULL)) { 1729 1667 INIT_LIST_HEAD(&vma->closed_link); 1730 - __i915_vma_put(vma); 1668 + i915_vma_destroy(vma); 1731 1669 i915_gem_object_unlock(obj); 1732 1670 } else { 1733 1671 /* back you go.. */

+3

drivers/gpu/drm/i915/i915_vma.h

··· 236 236 kref_put(&vma->ref, i915_vma_release); 237 237 } 238 238 239 + void i915_vma_destroy_locked(struct i915_vma *vma); 240 + void i915_vma_destroy(struct i915_vma *vma); 241 + 239 242 #define assert_vma_held(vma) dma_resv_assert_held((vma)->obj->base.resv) 240 243 241 244 static inline void i915_vma_lock(struct i915_vma *vma)

+7 -19

drivers/gpu/drm/i915/intel_device_info.c

··· 170 170 INTEL_ICL_PORT_F_IDS(0), 171 171 }; 172 172 173 + static const u16 subplatform_uy_ids[] = { 174 + INTEL_TGL_12_GT2_IDS(0), 175 + }; 176 + 173 177 static const u16 subplatform_n_ids[] = { 174 178 INTEL_ADLN_IDS(0), 175 179 }; ··· 218 214 } else if (find_devid(devid, subplatform_portf_ids, 219 215 ARRAY_SIZE(subplatform_portf_ids))) { 220 216 mask = BIT(INTEL_SUBPLATFORM_PORTF); 217 + } else if (find_devid(devid, subplatform_uy_ids, 218 + ARRAY_SIZE(subplatform_uy_ids))) { 219 + mask = BIT(INTEL_SUBPLATFORM_UY); 221 220 } else if (find_devid(devid, subplatform_n_ids, 222 221 ARRAY_SIZE(subplatform_n_ids))) { 223 222 mask = BIT(INTEL_SUBPLATFORM_N); 224 223 } else if (find_devid(devid, subplatform_rpls_ids, 225 224 ARRAY_SIZE(subplatform_rpls_ids))) { 226 225 mask = BIT(INTEL_SUBPLATFORM_RPL_S); 227 - } 228 - 229 - if (IS_TIGERLAKE(i915)) { 230 - struct pci_dev *root, *pdev = to_pci_dev(i915->drm.dev); 231 - 232 - root = list_first_entry(&pdev->bus->devices, typeof(*root), bus_list); 233 - 234 - drm_WARN_ON(&i915->drm, mask); 235 - drm_WARN_ON(&i915->drm, (root->device & TGL_ROOT_DEVICE_MASK) != 236 - TGL_ROOT_DEVICE_ID); 237 - 238 - switch (root->device & TGL_ROOT_DEVICE_SKU_MASK) { 239 - case TGL_ROOT_DEVICE_SKU_ULX: 240 - mask = BIT(INTEL_SUBPLATFORM_ULX); 241 - break; 242 - case TGL_ROOT_DEVICE_SKU_ULT: 243 - mask = BIT(INTEL_SUBPLATFORM_ULT); 244 - break; 245 - } 246 226 } 247 227 248 228 GEM_BUG_ON(mask & ~INTEL_SUBPLATFORM_MASK);

+5

drivers/gpu/drm/i915/intel_device_info.h

··· 106 106 /* ICL */ 107 107 #define INTEL_SUBPLATFORM_PORTF (0) 108 108 109 + /* TGL */ 110 + #define INTEL_SUBPLATFORM_UY (0) 111 + 109 112 /* DG2 */ 110 113 #define INTEL_SUBPLATFORM_G10 0 111 114 #define INTEL_SUBPLATFORM_G11 1 ··· 134 131 /* Keep has_* in alphabetical order */ \ 135 132 func(has_64bit_reloc); \ 136 133 func(has_64k_pages); \ 134 + func(needs_compact_pt); \ 137 135 func(gpu_reset_clobbers_display); \ 138 136 func(has_reset_engine); \ 137 + func(has_flat_ccs); \ 139 138 func(has_global_mocs); \ 140 139 func(has_gt_uc); \ 141 140 func(has_guc_deprivilege); \

+8 -2

drivers/gpu/drm/i915/intel_memory_region.c

··· 97 97 bool test_all, 98 98 const void *caller) 99 99 { 100 - resource_size_t last = resource_size(&mem->region) - PAGE_SIZE; 101 - resource_size_t page; 100 + resource_size_t last, page; 102 101 int err; 102 + 103 + if (mem->io_size < PAGE_SIZE) 104 + return 0; 105 + 106 + last = mem->io_size - PAGE_SIZE; 103 107 104 108 /* 105 109 * Quick test to check read/write access to the iomap (backing store). ··· 221 217 resource_size_t size, 222 218 resource_size_t min_page_size, 223 219 resource_size_t io_start, 220 + resource_size_t io_size, 224 221 u16 type, 225 222 u16 instance, 226 223 const struct intel_memory_region_ops *ops) ··· 236 231 mem->i915 = i915; 237 232 mem->region = (struct resource)DEFINE_RES_MEM(start, size); 238 233 mem->io_start = io_start; 234 + mem->io_size = io_size; 239 235 mem->min_page_size = min_page_size; 240 236 mem->ops = ops; 241 237 mem->total = size;

+2 -5

drivers/gpu/drm/i915/intel_memory_region.h

··· 67 67 struct io_mapping iomap; 68 68 struct resource region; 69 69 70 - /* For fake LMEM */ 71 - struct drm_mm_node fake_mappable; 72 - 73 70 resource_size_t io_start; 71 + resource_size_t io_size; 74 72 resource_size_t min_page_size; 75 73 resource_size_t total; 76 74 resource_size_t avail; ··· 78 80 enum intel_region_id id; 79 81 char name[16]; 80 82 bool private; /* not for userspace */ 81 - 82 - dma_addr_t remap_addr; 83 83 84 84 struct { 85 85 struct mutex lock; /* Protects access to objects */ ··· 99 103 resource_size_t size, 100 104 resource_size_t min_page_size, 101 105 resource_size_t io_start, 106 + resource_size_t io_size, 102 107 u16 type, 103 108 u16 instance, 104 109 const struct intel_memory_region_ops *ops);

+20 -1

drivers/gpu/drm/i915/intel_region_ttm.c

··· 87 87 88 88 ret = i915_ttm_buddy_man_init(bdev, mem_type, false, 89 89 resource_size(&mem->region), 90 + mem->io_size, 90 91 mem->min_page_size, PAGE_SIZE); 91 92 if (ret) 92 93 return ret; ··· 200 199 struct ttm_resource *res; 201 200 int ret; 202 201 202 + if (flags & I915_BO_ALLOC_CONTIGUOUS) 203 + place.flags |= TTM_PL_FLAG_CONTIGUOUS; 204 + if (mem->io_size && mem->io_size < mem->total) { 205 + if (flags & I915_BO_ALLOC_GPU_ONLY) { 206 + place.flags |= TTM_PL_FLAG_TOPDOWN; 207 + } else { 208 + place.fpfn = 0; 209 + place.lpfn = mem->io_size >> PAGE_SHIFT; 210 + } 211 + } 212 + 203 213 mock_bo.base.size = size; 204 - place.flags = flags; 214 + mock_bo.bdev = &mem->i915->bdev; 205 215 206 216 ret = man->func->alloc(man, &mock_bo, &place, &res); 207 217 if (ret == -ENOSPC) 208 218 ret = -ENXIO; 219 + if (!ret) 220 + res->bo = NULL; /* Rather blow up, then some uaf */ 209 221 return ret ? ERR_PTR(ret) : res; 210 222 } 211 223 ··· 233 219 struct ttm_resource *res) 234 220 { 235 221 struct ttm_resource_manager *man = mem->region_private; 222 + struct ttm_buffer_object mock_bo = {}; 223 + 224 + mock_bo.base.size = res->num_pages << PAGE_SHIFT; 225 + mock_bo.bdev = &mem->i915->bdev; 226 + res->bo = &mock_bo; 236 227 237 228 man->func->free(man, res); 238 229 }

+1 -1

drivers/gpu/drm/i915/intel_step.c

··· 165 165 } else if (IS_ROCKETLAKE(i915)) { 166 166 revids = rkl_revids; 167 167 size = ARRAY_SIZE(rkl_revids); 168 - } else if (IS_TGL_U(i915) || IS_TGL_Y(i915)) { 168 + } else if (IS_TGL_UY(i915)) { 169 169 revids = tgl_uy_revids; 170 170 size = ARRAY_SIZE(tgl_uy_revids); 171 171 } else if (IS_TIGERLAKE(i915)) {

+189 -33

drivers/gpu/drm/i915/selftests/i915_gem_gtt.c

··· 27 27 28 28 #include "gem/i915_gem_context.h" 29 29 #include "gem/i915_gem_internal.h" 30 + #include "gem/i915_gem_region.h" 30 31 #include "gem/selftests/mock_context.h" 31 32 #include "gt/intel_context.h" 32 33 #include "gt/intel_gpu_commands.h" 34 + #include "gt/intel_gtt.h" 33 35 34 36 #include "i915_random.h" 35 37 #include "i915_selftest.h" ··· 241 239 u64 hole_start, u64 hole_end, 242 240 unsigned long end_time) 243 241 { 242 + const unsigned int min_alignment = 243 + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 244 244 I915_RND_STATE(seed_prng); 245 245 struct i915_vma_resource *mock_vma_res; 246 246 unsigned int size; ··· 256 252 I915_RND_SUBSTATE(prng, seed_prng); 257 253 struct drm_i915_gem_object *obj; 258 254 unsigned int *order, count, n; 259 - u64 hole_size; 255 + u64 hole_size, aligned_size; 260 256 261 - hole_size = (hole_end - hole_start) >> size; 257 + aligned_size = max_t(u32, ilog2(min_alignment), size); 258 + hole_size = (hole_end - hole_start) >> aligned_size; 262 259 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 263 260 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 264 261 count = hole_size >> 1; ··· 280 275 } 281 276 GEM_BUG_ON(!order); 282 277 283 - GEM_BUG_ON(count * BIT_ULL(size) > vm->total); 284 - GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); 278 + GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); 279 + GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); 285 280 286 281 /* Ignore allocation failures (i.e. don't report them as 287 282 * a test failure) as we are purposefully allocating very ··· 304 299 } 305 300 306 301 for (n = 0; n < count; n++) { 307 - u64 addr = hole_start + order[n] * BIT_ULL(size); 302 + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 308 303 intel_wakeref_t wakeref; 309 304 310 - GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 305 + GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); 311 306 312 307 if (igt_timeout(end_time, 313 308 "%s timed out before %d/%d\n", ··· 350 345 } 351 346 352 347 mock_vma_res->bi.pages = obj->mm.pages; 353 - mock_vma_res->node_size = BIT_ULL(size); 348 + mock_vma_res->node_size = BIT_ULL(aligned_size); 354 349 mock_vma_res->start = addr; 355 350 356 351 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) ··· 361 356 362 357 i915_random_reorder(order, count, &prng); 363 358 for (n = 0; n < count; n++) { 364 - u64 addr = hole_start + order[n] * BIT_ULL(size); 359 + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 365 360 intel_wakeref_t wakeref; 366 361 367 362 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); ··· 405 400 { 406 401 const u64 hole_size = hole_end - hole_start; 407 402 struct drm_i915_gem_object *obj; 403 + const unsigned int min_alignment = 404 + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 408 405 const unsigned long max_pages = 409 - min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); 406 + min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); 410 407 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 411 408 unsigned long npages, prime, flags; 412 409 struct i915_vma *vma; ··· 449 442 450 443 offset = p->offset; 451 444 list_for_each_entry(obj, &objects, st_link) { 445 + u64 aligned_size = round_up(obj->base.size, 446 + min_alignment); 447 + 452 448 vma = i915_vma_instance(obj, vm, NULL); 453 449 if (IS_ERR(vma)) 454 450 continue; 455 451 456 452 if (p->step < 0) { 457 - if (offset < hole_start + obj->base.size) 453 + if (offset < hole_start + aligned_size) 458 454 break; 459 - offset -= obj->base.size; 455 + offset -= aligned_size; 460 456 } 461 457 462 458 err = i915_vma_pin(vma, 0, 0, offset | flags); ··· 481 471 i915_vma_unpin(vma); 482 472 483 473 if (p->step > 0) { 484 - if (offset + obj->base.size > hole_end) 474 + if (offset + aligned_size > hole_end) 485 475 break; 486 - offset += obj->base.size; 476 + offset += aligned_size; 487 477 } 488 478 } 489 479 490 480 offset = p->offset; 491 481 list_for_each_entry(obj, &objects, st_link) { 482 + u64 aligned_size = round_up(obj->base.size, 483 + min_alignment); 484 + 492 485 vma = i915_vma_instance(obj, vm, NULL); 493 486 if (IS_ERR(vma)) 494 487 continue; 495 488 496 489 if (p->step < 0) { 497 - if (offset < hole_start + obj->base.size) 490 + if (offset < hole_start + aligned_size) 498 491 break; 499 - offset -= obj->base.size; 492 + offset -= aligned_size; 500 493 } 501 494 502 495 if (!drm_mm_node_allocated(&vma->node) || ··· 520 507 } 521 508 522 509 if (p->step > 0) { 523 - if (offset + obj->base.size > hole_end) 510 + if (offset + aligned_size > hole_end) 524 511 break; 525 - offset += obj->base.size; 512 + offset += aligned_size; 526 513 } 527 514 } 528 515 529 516 offset = p->offset; 530 517 list_for_each_entry_reverse(obj, &objects, st_link) { 518 + u64 aligned_size = round_up(obj->base.size, 519 + min_alignment); 520 + 531 521 vma = i915_vma_instance(obj, vm, NULL); 532 522 if (IS_ERR(vma)) 533 523 continue; 534 524 535 525 if (p->step < 0) { 536 - if (offset < hole_start + obj->base.size) 526 + if (offset < hole_start + aligned_size) 537 527 break; 538 - offset -= obj->base.size; 528 + offset -= aligned_size; 539 529 } 540 530 541 531 err = i915_vma_pin(vma, 0, 0, offset | flags); ··· 560 544 i915_vma_unpin(vma); 561 545 562 546 if (p->step > 0) { 563 - if (offset + obj->base.size > hole_end) 547 + if (offset + aligned_size > hole_end) 564 548 break; 565 - offset += obj->base.size; 549 + offset += aligned_size; 566 550 } 567 551 } 568 552 569 553 offset = p->offset; 570 554 list_for_each_entry_reverse(obj, &objects, st_link) { 555 + u64 aligned_size = round_up(obj->base.size, 556 + min_alignment); 557 + 571 558 vma = i915_vma_instance(obj, vm, NULL); 572 559 if (IS_ERR(vma)) 573 560 continue; 574 561 575 562 if (p->step < 0) { 576 - if (offset < hole_start + obj->base.size) 563 + if (offset < hole_start + aligned_size) 577 564 break; 578 - offset -= obj->base.size; 565 + offset -= aligned_size; 579 566 } 580 567 581 568 if (!drm_mm_node_allocated(&vma->node) || ··· 599 580 } 600 581 601 582 if (p->step > 0) { 602 - if (offset + obj->base.size > hole_end) 583 + if (offset + aligned_size > hole_end) 603 584 break; 604 - offset += obj->base.size; 585 + offset += aligned_size; 605 586 } 606 587 } 607 588 } ··· 631 612 const u64 hole_size = hole_end - hole_start; 632 613 const unsigned long max_pages = 633 614 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 615 + unsigned long min_alignment; 634 616 unsigned long flags; 635 617 u64 size; 636 618 ··· 640 620 flags = PIN_OFFSET_FIXED | PIN_USER; 641 621 if (i915_is_ggtt(vm)) 642 622 flags |= PIN_GLOBAL; 623 + 624 + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 643 625 644 626 for_each_prime_number_from(size, 1, max_pages) { 645 627 struct drm_i915_gem_object *obj; ··· 661 639 662 640 for (addr = hole_start; 663 641 addr + obj->base.size < hole_end; 664 - addr += obj->base.size) { 642 + addr += round_up(obj->base.size, min_alignment)) { 665 643 err = i915_vma_pin(vma, 0, 0, addr | flags); 666 644 if (err) { 667 645 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", ··· 713 691 { 714 692 struct drm_i915_gem_object *obj; 715 693 struct i915_vma *vma; 694 + unsigned int min_alignment; 716 695 unsigned long flags; 717 696 unsigned int pot; 718 697 int err = 0; ··· 721 698 flags = PIN_OFFSET_FIXED | PIN_USER; 722 699 if (i915_is_ggtt(vm)) 723 700 flags |= PIN_GLOBAL; 701 + 702 + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 724 703 725 704 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 726 705 if (IS_ERR(obj)) ··· 736 711 737 712 /* Insert a pair of pages across every pot boundary within the hole */ 738 713 for (pot = fls64(hole_end - 1) - 1; 739 - pot > ilog2(2 * I915_GTT_PAGE_SIZE); 714 + pot > ilog2(2 * min_alignment); 740 715 pot--) { 741 716 u64 step = BIT_ULL(pot); 742 717 u64 addr; 743 718 744 - for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 745 - addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; 719 + for (addr = round_up(hole_start + min_alignment, step) - min_alignment; 720 + addr <= round_down(hole_end - (2 * min_alignment), step) - min_alignment; 746 721 addr += step) { 747 722 err = i915_vma_pin(vma, 0, 0, addr | flags); 748 723 if (err) { ··· 787 762 unsigned long end_time) 788 763 { 789 764 I915_RND_STATE(prng); 765 + unsigned int min_alignment; 790 766 unsigned int size; 791 767 unsigned long flags; 792 768 ··· 795 769 if (i915_is_ggtt(vm)) 796 770 flags |= PIN_GLOBAL; 797 771 772 + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 773 + 798 774 /* Keep creating larger objects until one cannot fit into the hole */ 799 775 for (size = 12; (hole_end - hole_start) >> size; size++) { 800 776 struct drm_i915_gem_object *obj; 801 777 unsigned int *order, count, n; 802 778 struct i915_vma *vma; 803 - u64 hole_size; 779 + u64 hole_size, aligned_size; 804 780 int err = -ENODEV; 805 781 806 - hole_size = (hole_end - hole_start) >> size; 782 + aligned_size = max_t(u32, ilog2(min_alignment), size); 783 + hole_size = (hole_end - hole_start) >> aligned_size; 807 784 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 808 785 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 809 786 count = hole_size >> 1; ··· 846 817 GEM_BUG_ON(vma->size != BIT_ULL(size)); 847 818 848 819 for (n = 0; n < count; n++) { 849 - u64 addr = hole_start + order[n] * BIT_ULL(size); 820 + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 850 821 851 822 err = i915_vma_pin(vma, 0, 0, addr | flags); 852 823 if (err) { ··· 898 869 { 899 870 struct drm_i915_gem_object *obj; 900 871 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 872 + unsigned int min_alignment; 901 873 unsigned int order = 12; 902 874 LIST_HEAD(objects); 903 875 int err = 0; 904 876 u64 addr; 877 + 878 + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 905 879 906 880 /* Keep creating larger objects until one cannot fit into the hole */ 907 881 for (addr = hole_start; addr < hole_end; ) { ··· 946 914 } 947 915 948 916 i915_vma_unpin(vma); 949 - addr += size; 917 + addr += round_up(size, min_alignment); 950 918 951 919 /* 952 920 * Since we are injecting allocation faults at random intervals, ··· 1070 1038 return err; 1071 1039 } 1072 1040 1041 + static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, 1042 + u64 addr, u64 size, unsigned long flags) 1043 + { 1044 + struct drm_i915_gem_object *obj; 1045 + struct i915_vma *vma; 1046 + int err = 0; 1047 + u64 expected_vma_size, expected_node_size; 1048 + bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || 1049 + mr->type == INTEL_MEMORY_STOLEN_LOCAL; 1050 + 1051 + obj = i915_gem_object_create_region(mr, size, 0, 0); 1052 + if (IS_ERR(obj)) { 1053 + /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ 1054 + if (PTR_ERR(obj) == -ENODEV && is_stolen) 1055 + return 0; 1056 + return PTR_ERR(obj); 1057 + } 1058 + 1059 + vma = i915_vma_instance(obj, vm, NULL); 1060 + if (IS_ERR(vma)) { 1061 + err = PTR_ERR(vma); 1062 + goto err_put; 1063 + } 1064 + 1065 + err = i915_vma_pin(vma, 0, 0, addr | flags); 1066 + if (err) 1067 + goto err_put; 1068 + i915_vma_unpin(vma); 1069 + 1070 + if (!drm_mm_node_allocated(&vma->node)) { 1071 + err = -EINVAL; 1072 + goto err_put; 1073 + } 1074 + 1075 + if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { 1076 + err = -EINVAL; 1077 + goto err_put; 1078 + } 1079 + 1080 + expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); 1081 + expected_node_size = expected_vma_size; 1082 + 1083 + if (NEEDS_COMPACT_PT(vm->i915) && i915_gem_object_is_lmem(obj)) { 1084 + /* compact-pt should expand lmem node to 2MB */ 1085 + expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1086 + expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1087 + } 1088 + 1089 + if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { 1090 + err = i915_vma_unbind_unlocked(vma); 1091 + err = -EBADSLT; 1092 + goto err_put; 1093 + } 1094 + 1095 + err = i915_vma_unbind_unlocked(vma); 1096 + if (err) 1097 + goto err_put; 1098 + 1099 + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1100 + 1101 + err_put: 1102 + i915_gem_object_put(obj); 1103 + cleanup_freed_objects(vm->i915); 1104 + return err; 1105 + } 1106 + 1107 + static int misaligned_pin(struct i915_address_space *vm, 1108 + u64 hole_start, u64 hole_end, 1109 + unsigned long end_time) 1110 + { 1111 + struct intel_memory_region *mr; 1112 + enum intel_region_id id; 1113 + unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 1114 + int err = 0; 1115 + u64 hole_size = hole_end - hole_start; 1116 + 1117 + if (i915_is_ggtt(vm)) 1118 + flags |= PIN_GLOBAL; 1119 + 1120 + for_each_memory_region(mr, vm->i915, id) { 1121 + u64 min_alignment = i915_vm_min_alignment(vm, (enum intel_memory_type)id); 1122 + u64 size = min_alignment; 1123 + u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); 1124 + 1125 + /* avoid -ENOSPC on very small hole setups */ 1126 + if (hole_size < 3 * min_alignment) 1127 + continue; 1128 + 1129 + /* we can't test < 4k alignment due to flags being encoded in lower bits */ 1130 + if (min_alignment != I915_GTT_PAGE_SIZE_4K) { 1131 + err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); 1132 + /* misaligned should error with -EINVAL*/ 1133 + if (!err) 1134 + err = -EBADSLT; 1135 + if (err != -EINVAL) 1136 + return err; 1137 + } 1138 + 1139 + /* test for vma->size expansion to min page size */ 1140 + err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); 1141 + if (err) 1142 + return err; 1143 + 1144 + /* test for intermediate size not expanding vma->size for large alignments */ 1145 + err = misaligned_case(vm, mr, addr, size / 2, flags); 1146 + if (err) 1147 + return err; 1148 + } 1149 + 1150 + return 0; 1151 + } 1152 + 1073 1153 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 1074 1154 int (*func)(struct i915_address_space *vm, 1075 1155 u64 hole_start, u64 hole_end, ··· 1249 1105 static int igt_ppgtt_shrink_boom(void *arg) 1250 1106 { 1251 1107 return exercise_ppgtt(arg, shrink_boom); 1108 + } 1109 + 1110 + static int igt_ppgtt_misaligned_pin(void *arg) 1111 + { 1112 + return exercise_ppgtt(arg, misaligned_pin); 1252 1113 } 1253 1114 1254 1115 static int sort_holes(void *priv, const struct list_head *A, ··· 1326 1177 static int igt_ggtt_lowlevel(void *arg) 1327 1178 { 1328 1179 return exercise_ggtt(arg, lowlevel_hole); 1180 + } 1181 + 1182 + static int igt_ggtt_misaligned_pin(void *arg) 1183 + { 1184 + return exercise_ggtt(arg, misaligned_pin); 1329 1185 } 1330 1186 1331 1187 static int igt_ggtt_page(void *arg) ··· 2305 2151 SUBTEST(igt_ppgtt_fill), 2306 2152 SUBTEST(igt_ppgtt_shrink), 2307 2153 SUBTEST(igt_ppgtt_shrink_boom), 2154 + SUBTEST(igt_ppgtt_misaligned_pin), 2308 2155 SUBTEST(igt_ggtt_lowlevel), 2309 2156 SUBTEST(igt_ggtt_drunk), 2310 2157 SUBTEST(igt_ggtt_walk), 2311 2158 SUBTEST(igt_ggtt_pot), 2312 2159 SUBTEST(igt_ggtt_fill), 2313 2160 SUBTEST(igt_ggtt_page), 2161 + SUBTEST(igt_ggtt_misaligned_pin), 2314 2162 SUBTEST(igt_cs_tlb), 2315 2163 }; 2316 2164

+154 -5

drivers/gpu/drm/i915/selftests/intel_memory_region.c

··· 17 17 #include "gem/i915_gem_context.h" 18 18 #include "gem/i915_gem_lmem.h" 19 19 #include "gem/i915_gem_region.h" 20 + #include "gem/i915_gem_ttm.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 21 22 #include "gem/selftests/mock_context.h" 22 23 #include "gt/intel_engine_pm.h" ··· 171 170 if (!order) 172 171 return 0; 173 172 174 - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 173 + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); 175 174 if (IS_ERR(mem)) { 176 175 pr_err("failed to create memory region\n"); 177 176 err = PTR_ERR(mem); ··· 384 383 */ 385 384 386 385 size = (SZ_4G - 1) & PAGE_MASK; 387 - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); 386 + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); 388 387 if (IS_ERR(mem)) 389 388 return PTR_ERR(mem); 390 389 ··· 472 471 */ 473 472 474 473 size = SZ_8G; 475 - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); 474 + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); 476 475 if (IS_ERR(mem)) 477 476 return PTR_ERR(mem); 478 477 ··· 510 509 close_objects(mem, &objects); 511 510 out_put: 512 511 intel_memory_region_destroy(mem); 512 + return err; 513 + } 514 + 515 + static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj) 516 + { 517 + struct intel_memory_region *mr = obj->mm.region; 518 + struct i915_ttm_buddy_resource *bman_res = 519 + to_ttm_buddy_resource(obj->mm.res); 520 + struct drm_buddy *mm = bman_res->mm; 521 + struct drm_buddy_block *block; 522 + u64 total; 523 + 524 + total = 0; 525 + list_for_each_entry(block, &bman_res->blocks, link) { 526 + u64 start = drm_buddy_block_offset(block); 527 + u64 end = start + drm_buddy_block_size(mm, block); 528 + 529 + if (start < mr->io_size) 530 + total += min_t(u64, end, mr->io_size) - start; 531 + } 532 + 533 + return total; 534 + } 535 + 536 + static int igt_mock_io_size(void *arg) 537 + { 538 + struct intel_memory_region *mr = arg; 539 + struct drm_i915_private *i915 = mr->i915; 540 + struct drm_i915_gem_object *obj; 541 + u64 mappable_theft_total; 542 + u64 io_size; 543 + u64 total; 544 + u64 ps; 545 + u64 rem; 546 + u64 size; 547 + I915_RND_STATE(prng); 548 + LIST_HEAD(objects); 549 + int err = 0; 550 + 551 + ps = SZ_4K; 552 + if (i915_prandom_u64_state(&prng) & 1) 553 + ps = SZ_64K; /* For something like DG2 */ 554 + 555 + div64_u64_rem(i915_prandom_u64_state(&prng), SZ_8G, &total); 556 + total = round_down(total, ps); 557 + total = max_t(u64, total, SZ_1G); 558 + 559 + div64_u64_rem(i915_prandom_u64_state(&prng), total - ps, &io_size); 560 + io_size = round_down(io_size, ps); 561 + io_size = max_t(u64, io_size, SZ_256M); /* 256M seems to be the common lower limit */ 562 + 563 + pr_info("%s with ps=%llx, io_size=%llx, total=%llx\n", 564 + __func__, ps, io_size, total); 565 + 566 + mr = mock_region_create(i915, 0, total, ps, 0, io_size); 567 + if (IS_ERR(mr)) { 568 + err = PTR_ERR(mr); 569 + goto out_err; 570 + } 571 + 572 + mappable_theft_total = 0; 573 + rem = total - io_size; 574 + do { 575 + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); 576 + size = round_down(size, ps); 577 + size = max(size, ps); 578 + 579 + obj = igt_object_create(mr, &objects, size, 580 + I915_BO_ALLOC_GPU_ONLY); 581 + if (IS_ERR(obj)) { 582 + pr_err("%s TOPDOWN failed with rem=%llx, size=%llx\n", 583 + __func__, rem, size); 584 + err = PTR_ERR(obj); 585 + goto out_close; 586 + } 587 + 588 + mappable_theft_total += igt_object_mappable_total(obj); 589 + rem -= size; 590 + } while (rem); 591 + 592 + pr_info("%s mappable theft=(%lluMiB/%lluMiB), total=%lluMiB\n", 593 + __func__, 594 + (u64)mappable_theft_total >> 20, 595 + (u64)io_size >> 20, 596 + (u64)total >> 20); 597 + 598 + /* 599 + * Even if we allocate all of the non-mappable portion, we should still 600 + * be able to dip into the mappable portion. 601 + */ 602 + obj = igt_object_create(mr, &objects, io_size, 603 + I915_BO_ALLOC_GPU_ONLY); 604 + if (IS_ERR(obj)) { 605 + pr_err("%s allocation unexpectedly failed\n", __func__); 606 + err = PTR_ERR(obj); 607 + goto out_close; 608 + } 609 + 610 + close_objects(mr, &objects); 611 + 612 + rem = io_size; 613 + do { 614 + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); 615 + size = round_down(size, ps); 616 + size = max(size, ps); 617 + 618 + obj = igt_object_create(mr, &objects, size, 0); 619 + if (IS_ERR(obj)) { 620 + pr_err("%s MAPPABLE failed with rem=%llx, size=%llx\n", 621 + __func__, rem, size); 622 + err = PTR_ERR(obj); 623 + goto out_close; 624 + } 625 + 626 + if (igt_object_mappable_total(obj) != size) { 627 + pr_err("%s allocation is not mappable(size=%llx)\n", 628 + __func__, size); 629 + err = -EINVAL; 630 + goto out_close; 631 + } 632 + rem -= size; 633 + } while (rem); 634 + 635 + /* 636 + * We assume CPU access is required by default, which should result in a 637 + * failure here, even though the non-mappable portion is free. 638 + */ 639 + obj = igt_object_create(mr, &objects, ps, 0); 640 + if (!IS_ERR(obj)) { 641 + pr_err("%s allocation unexpectedly succeeded\n", __func__); 642 + err = -EINVAL; 643 + goto out_close; 644 + } 645 + 646 + out_close: 647 + close_objects(mr, &objects); 648 + intel_memory_region_destroy(mr); 649 + out_err: 650 + if (err == -ENOMEM) 651 + err = 0; 652 + 513 653 return err; 514 654 } 515 655 ··· 822 680 823 681 i915_gem_object_lock(obj, NULL); 824 682 err = i915_gem_object_pin_pages(obj); 825 - if (err) 683 + if (err) { 684 + if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) { 685 + pr_info("%s not enough lmem for ps(%u) err=%d\n", 686 + __func__, ps, err); 687 + err = 0; 688 + } 826 689 goto out_put; 690 + } 827 691 828 692 daddr = i915_gem_object_get_dma_address(obj, 0); 829 693 if (!IS_ALIGNED(daddr, ps)) { ··· 1327 1179 SUBTEST(igt_mock_contiguous), 1328 1180 SUBTEST(igt_mock_splintered_region), 1329 1181 SUBTEST(igt_mock_max_segment), 1182 + SUBTEST(igt_mock_io_size), 1330 1183 }; 1331 1184 struct intel_memory_region *mem; 1332 1185 struct drm_i915_private *i915; ··· 1337 1188 if (!i915) 1338 1189 return -ENOMEM; 1339 1190 1340 - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); 1191 + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); 1341 1192 if (IS_ERR(mem)) { 1342 1193 pr_err("failed to create memory region\n"); 1343 1194 err = PTR_ERR(mem);

+5 -8

drivers/gpu/drm/i915/selftests/mock_region.c

··· 22 22 23 23 static int mock_region_get_pages(struct drm_i915_gem_object *obj) 24 24 { 25 - unsigned int flags; 26 25 struct sg_table *pages; 27 26 int err; 28 27 29 - flags = 0; 30 - if (obj->flags & I915_BO_ALLOC_CONTIGUOUS) 31 - flags |= TTM_PL_FLAG_CONTIGUOUS; 32 - 33 28 obj->mm.res = intel_region_ttm_resource_alloc(obj->mm.region, 34 29 obj->base.size, 35 - flags); 30 + obj->flags); 36 31 if (IS_ERR(obj->mm.res)) 37 32 return PTR_ERR(obj->mm.res); 38 33 ··· 102 107 resource_size_t start, 103 108 resource_size_t size, 104 109 resource_size_t min_page_size, 105 - resource_size_t io_start) 110 + resource_size_t io_start, 111 + resource_size_t io_size) 106 112 { 107 113 int instance = ida_alloc_max(&i915->selftest.mock_region_instances, 108 114 TTM_NUM_MEM_TYPES - TTM_PL_PRIV - 1, ··· 113 117 return ERR_PTR(instance); 114 118 115 119 return intel_memory_region_create(i915, start, size, min_page_size, 116 - io_start, INTEL_MEMORY_MOCK, instance, 120 + io_start, io_size, 121 + INTEL_MEMORY_MOCK, instance, 117 122 &mock_region_ops); 118 123 }

+2 -1

drivers/gpu/drm/i915/selftests/mock_region.h

··· 16 16 resource_size_t start, 17 17 resource_size_t size, 18 18 resource_size_t min_page_size, 19 - resource_size_t io_start); 19 + resource_size_t io_start, 20 + resource_size_t io_size); 20 21 21 22 #endif /* !__MOCK_REGION_H */

+40 -5

include/uapi/drm/i915_drm.h

··· 1118 1118 /** 1119 1119 * When the EXEC_OBJECT_PINNED flag is specified this is populated by 1120 1120 * the user with the GTT offset at which this object will be pinned. 1121 + * 1121 1122 * When the I915_EXEC_NO_RELOC flag is specified this must contain the 1122 1123 * presumed_offset of the object. 1124 + * 1123 1125 * During execbuffer2 the kernel populates it with the value of the 1124 1126 * current GTT offset of the object, for future presumed_offset writes. 1127 + * 1128 + * See struct drm_i915_gem_create_ext for the rules when dealing with 1129 + * alignment restrictions with I915_MEMORY_CLASS_DEVICE, on devices with 1130 + * minimum page sizes, like DG2. 1125 1131 */ 1126 1132 __u64 offset; 1127 1133 ··· 3150 3144 * 3151 3145 * The (page-aligned) allocated size for the object will be returned. 3152 3146 * 3153 - * Note that for some devices we have might have further minimum 3154 - * page-size restrictions(larger than 4K), like for device local-memory. 3155 - * However in general the final size here should always reflect any 3156 - * rounding up, if for example using the I915_GEM_CREATE_EXT_MEMORY_REGIONS 3157 - * extension to place the object in device local-memory. 3147 + * 3148 + * DG2 64K min page size implications: 3149 + * 3150 + * On discrete platforms, starting from DG2, we have to contend with GTT 3151 + * page size restrictions when dealing with I915_MEMORY_CLASS_DEVICE 3152 + * objects. Specifically the hardware only supports 64K or larger GTT 3153 + * page sizes for such memory. The kernel will already ensure that all 3154 + * I915_MEMORY_CLASS_DEVICE memory is allocated using 64K or larger page 3155 + * sizes underneath. 3156 + * 3157 + * Note that the returned size here will always reflect any required 3158 + * rounding up done by the kernel, i.e 4K will now become 64K on devices 3159 + * such as DG2. 3160 + * 3161 + * Special DG2 GTT address alignment requirement: 3162 + * 3163 + * The GTT alignment will also need to be at least 2M for such objects. 3164 + * 3165 + * Note that due to how the hardware implements 64K GTT page support, we 3166 + * have some further complications: 3167 + * 3168 + * 1) The entire PDE (which covers a 2MB virtual address range), must 3169 + * contain only 64K PTEs, i.e mixing 4K and 64K PTEs in the same 3170 + * PDE is forbidden by the hardware. 3171 + * 3172 + * 2) We still need to support 4K PTEs for I915_MEMORY_CLASS_SYSTEM 3173 + * objects. 3174 + * 3175 + * To keep things simple for userland, we mandate that any GTT mappings 3176 + * must be aligned to and rounded up to 2MB. The kernel will internally 3177 + * pad them out to the next 2MB boundary. As this only wastes virtual 3178 + * address space and avoids userland having to copy any needlessly 3179 + * complicated PDE sharing scheme (coloring) and only affects DG2, this 3180 + * is deemed to be a good compromise. 3158 3181 */ 3159 3182 __u64 size; 3160 3183 /**

Configure Feed

Configure Feed