Revert "drm/amdgpu: revert to old status lock handling v4"

+4 -4

drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c

··· 1040 1040 struct amdgpu_bo *bo; 1041 1041 int ret; 1042 1042 1043 - spin_lock(&vm->invalidated_lock); 1043 + spin_lock(&vm->status_lock); 1044 1044 while (!list_empty(&vm->invalidated)) { 1045 1045 bo_va = list_first_entry(&vm->invalidated, 1046 1046 struct amdgpu_bo_va, 1047 1047 base.vm_status); 1048 - spin_unlock(&vm->invalidated_lock); 1048 + spin_unlock(&vm->status_lock); 1049 1049 1050 1050 bo = bo_va->base.bo; 1051 1051 ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2); ··· 1062 1062 if (ret) 1063 1063 return ret; 1064 1064 1065 - spin_lock(&vm->invalidated_lock); 1065 + spin_lock(&vm->status_lock); 1066 1066 } 1067 - spin_unlock(&vm->invalidated_lock); 1067 + spin_unlock(&vm->status_lock); 1068 1068 1069 1069 return 0; 1070 1070 }

+91 -55

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

··· 153 153 154 154 vm_bo->moved = true; 155 155 amdgpu_vm_assert_locked(vm); 156 + spin_lock(&vm_bo->vm->status_lock); 156 157 if (bo->tbo.type == ttm_bo_type_kernel) 157 158 list_move(&vm_bo->vm_status, &vm->evicted); 158 159 else 159 160 list_move_tail(&vm_bo->vm_status, &vm->evicted); 161 + spin_unlock(&vm_bo->vm->status_lock); 160 162 } 161 163 /** 162 164 * amdgpu_vm_bo_moved - vm_bo is moved ··· 171 169 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 172 170 { 173 171 amdgpu_vm_assert_locked(vm_bo->vm); 172 + spin_lock(&vm_bo->vm->status_lock); 174 173 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 174 + spin_unlock(&vm_bo->vm->status_lock); 175 175 } 176 176 177 177 /** ··· 187 183 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 188 184 { 189 185 amdgpu_vm_assert_locked(vm_bo->vm); 186 + spin_lock(&vm_bo->vm->status_lock); 190 187 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 188 + spin_unlock(&vm_bo->vm->status_lock); 191 189 vm_bo->moved = false; 192 190 } 193 191 ··· 203 197 */ 204 198 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 205 199 { 206 - spin_lock(&vm_bo->vm->invalidated_lock); 200 + spin_lock(&vm_bo->vm->status_lock); 207 201 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 208 - spin_unlock(&vm_bo->vm->invalidated_lock); 202 + spin_unlock(&vm_bo->vm->status_lock); 209 203 } 210 204 211 205 /** ··· 218 212 */ 219 213 static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo) 220 214 { 221 - amdgpu_vm_assert_locked(vm_bo->vm); 222 215 vm_bo->moved = true; 216 + spin_lock(&vm_bo->vm->status_lock); 223 217 list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user); 218 + spin_unlock(&vm_bo->vm->status_lock); 224 219 } 225 220 226 221 /** ··· 235 228 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 236 229 { 237 230 amdgpu_vm_assert_locked(vm_bo->vm); 238 - if (vm_bo->bo->parent) 231 + if (vm_bo->bo->parent) { 232 + spin_lock(&vm_bo->vm->status_lock); 239 233 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 240 - else 234 + spin_unlock(&vm_bo->vm->status_lock); 235 + } else { 241 236 amdgpu_vm_bo_idle(vm_bo); 237 + } 242 238 } 243 239 244 240 /** ··· 255 245 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 256 246 { 257 247 amdgpu_vm_assert_locked(vm_bo->vm); 248 + spin_lock(&vm_bo->vm->status_lock); 258 249 list_move(&vm_bo->vm_status, &vm_bo->vm->done); 250 + spin_unlock(&vm_bo->vm->status_lock); 259 251 } 260 252 261 253 /** ··· 271 259 { 272 260 struct amdgpu_vm_bo_base *vm_bo, *tmp; 273 261 274 - spin_lock(&vm->invalidated_lock); 262 + amdgpu_vm_assert_locked(vm); 263 + 264 + spin_lock(&vm->status_lock); 275 265 list_splice_init(&vm->done, &vm->invalidated); 276 266 list_for_each_entry(vm_bo, &vm->invalidated, vm_status) 277 267 vm_bo->moved = true; 278 - spin_unlock(&vm->invalidated_lock); 279 268 280 - amdgpu_vm_assert_locked(vm); 281 269 list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) { 282 270 struct amdgpu_bo *bo = vm_bo->bo; 283 271 ··· 287 275 else if (bo->parent) 288 276 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 289 277 } 278 + spin_unlock(&vm->status_lock); 290 279 } 291 280 292 281 /** 293 282 * amdgpu_vm_update_shared - helper to update shared memory stat 294 283 * @base: base structure for tracking BO usage in a VM 295 284 * 296 - * Takes the vm stats_lock and updates the shared memory stat. If the basic 285 + * Takes the vm status_lock and updates the shared memory stat. If the basic 297 286 * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called 298 287 * as well. 299 288 */ ··· 307 294 bool shared; 308 295 309 296 dma_resv_assert_held(bo->tbo.base.resv); 310 - spin_lock(&vm->stats_lock); 297 + spin_lock(&vm->status_lock); 311 298 shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); 312 299 if (base->shared != shared) { 313 300 base->shared = shared; ··· 319 306 vm->stats[bo_memtype].drm.private += size; 320 307 } 321 308 } 322 - spin_unlock(&vm->stats_lock); 309 + spin_unlock(&vm->status_lock); 323 310 } 324 311 325 312 /** ··· 344 331 * be bo->tbo.resource 345 332 * @sign: if we should add (+1) or subtract (-1) from the stat 346 333 * 347 - * Caller need to have the vm stats_lock held. Useful for when multiple update 334 + * Caller need to have the vm status_lock held. Useful for when multiple update 348 335 * need to happen at the same time. 349 336 */ 350 337 static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, 351 - struct ttm_resource *res, int sign) 338 + struct ttm_resource *res, int sign) 352 339 { 353 340 struct amdgpu_vm *vm = base->vm; 354 341 struct amdgpu_bo *bo = base->bo; ··· 372 359 */ 373 360 if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) 374 361 vm->stats[res_memtype].drm.purgeable += size; 375 - if (!(bo->preferred_domains & 376 - amdgpu_mem_type_to_domain(res_memtype))) 362 + if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype))) 377 363 vm->stats[bo_memtype].evicted += size; 378 364 } 379 365 } ··· 391 379 { 392 380 struct amdgpu_vm *vm = base->vm; 393 381 394 - spin_lock(&vm->stats_lock); 382 + spin_lock(&vm->status_lock); 395 383 amdgpu_vm_update_stats_locked(base, res, sign); 396 - spin_unlock(&vm->stats_lock); 384 + spin_unlock(&vm->status_lock); 397 385 } 398 386 399 387 /** ··· 419 407 base->next = bo->vm_bo; 420 408 bo->vm_bo = base; 421 409 422 - spin_lock(&vm->stats_lock); 410 + spin_lock(&vm->status_lock); 423 411 base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); 424 412 amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1); 425 - spin_unlock(&vm->stats_lock); 413 + spin_unlock(&vm->status_lock); 426 414 427 415 if (!amdgpu_vm_is_bo_always_valid(vm, bo)) 428 416 return; ··· 481 469 int ret; 482 470 483 471 /* We can only trust prev->next while holding the lock */ 484 - spin_lock(&vm->invalidated_lock); 472 + spin_lock(&vm->status_lock); 485 473 while (!list_is_head(prev->next, &vm->done)) { 486 474 bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status); 487 475 488 476 bo = bo_va->base.bo; 489 477 if (bo) { 490 478 amdgpu_bo_ref(bo); 491 - spin_unlock(&vm->invalidated_lock); 479 + spin_unlock(&vm->status_lock); 492 480 493 481 ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1); 494 482 amdgpu_bo_unref(&bo); 495 483 if (unlikely(ret)) 496 484 return ret; 497 485 498 - spin_lock(&vm->invalidated_lock); 486 + spin_lock(&vm->status_lock); 499 487 } 500 488 prev = prev->next; 501 489 } 502 - spin_unlock(&vm->invalidated_lock); 490 + spin_unlock(&vm->status_lock); 503 491 504 492 return 0; 505 493 } ··· 595 583 void *param) 596 584 { 597 585 uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm); 598 - struct amdgpu_vm_bo_base *bo_base, *tmp; 586 + struct amdgpu_vm_bo_base *bo_base; 599 587 struct amdgpu_bo *bo; 600 588 int r; 601 589 ··· 608 596 return r; 609 597 } 610 598 611 - list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { 599 + spin_lock(&vm->status_lock); 600 + while (!list_empty(&vm->evicted)) { 601 + bo_base = list_first_entry(&vm->evicted, 602 + struct amdgpu_vm_bo_base, 603 + vm_status); 604 + spin_unlock(&vm->status_lock); 605 + 612 606 bo = bo_base->bo; 613 607 614 608 r = validate(param, bo); ··· 627 609 vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); 628 610 amdgpu_vm_bo_relocated(bo_base); 629 611 } 612 + spin_lock(&vm->status_lock); 630 613 } 614 + while (ticket && !list_empty(&vm->evicted_user)) { 615 + bo_base = list_first_entry(&vm->evicted_user, 616 + struct amdgpu_vm_bo_base, 617 + vm_status); 618 + spin_unlock(&vm->status_lock); 631 619 632 - if (ticket) { 633 - list_for_each_entry_safe(bo_base, tmp, &vm->evicted_user, 634 - vm_status) { 635 - bo = bo_base->bo; 636 - dma_resv_assert_held(bo->tbo.base.resv); 620 + bo = bo_base->bo; 621 + dma_resv_assert_held(bo->tbo.base.resv); 637 622 638 - r = validate(param, bo); 639 - if (r) 640 - return r; 623 + r = validate(param, bo); 624 + if (r) 625 + return r; 641 626 642 - amdgpu_vm_bo_invalidated(bo_base); 643 - } 627 + amdgpu_vm_bo_invalidated(bo_base); 628 + 629 + spin_lock(&vm->status_lock); 644 630 } 631 + spin_unlock(&vm->status_lock); 645 632 646 633 amdgpu_vm_eviction_lock(vm); 647 634 vm->evicting = false; ··· 675 652 ret = !vm->evicting; 676 653 amdgpu_vm_eviction_unlock(vm); 677 654 655 + spin_lock(&vm->status_lock); 678 656 ret &= list_empty(&vm->evicted); 657 + spin_unlock(&vm->status_lock); 679 658 680 659 spin_lock(&vm->immediate.lock); 681 660 ret &= !vm->immediate.stopped; ··· 971 946 struct amdgpu_vm *vm, bool immediate) 972 947 { 973 948 struct amdgpu_vm_update_params params; 974 - struct amdgpu_vm_bo_base *entry, *tmp; 949 + struct amdgpu_vm_bo_base *entry; 975 950 bool flush_tlb_needed = false; 951 + LIST_HEAD(relocated); 976 952 int r, idx; 977 953 978 954 amdgpu_vm_assert_locked(vm); 979 955 980 - if (list_empty(&vm->relocated)) 956 + spin_lock(&vm->status_lock); 957 + list_splice_init(&vm->relocated, &relocated); 958 + spin_unlock(&vm->status_lock); 959 + 960 + if (list_empty(&relocated)) 981 961 return 0; 982 962 983 963 if (!drm_dev_enter(adev_to_drm(adev), &idx)) ··· 998 968 if (r) 999 969 goto error; 1000 970 1001 - list_for_each_entry(entry, &vm->relocated, vm_status) { 971 + list_for_each_entry(entry, &relocated, vm_status) { 1002 972 /* vm_flush_needed after updating moved PDEs */ 1003 973 flush_tlb_needed |= entry->moved; 1004 974 ··· 1014 984 if (flush_tlb_needed) 1015 985 atomic64_inc(&vm->tlb_seq); 1016 986 1017 - list_for_each_entry_safe(entry, tmp, &vm->relocated, vm_status) { 987 + while (!list_empty(&relocated)) { 988 + entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, 989 + vm_status); 1018 990 amdgpu_vm_bo_idle(entry); 1019 991 } 1020 992 ··· 1243 1211 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, 1244 1212 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) 1245 1213 { 1246 - spin_lock(&vm->stats_lock); 1214 + spin_lock(&vm->status_lock); 1247 1215 memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM); 1248 - spin_unlock(&vm->stats_lock); 1216 + spin_unlock(&vm->status_lock); 1249 1217 } 1250 1218 1251 1219 /** ··· 1612 1580 struct amdgpu_vm *vm, 1613 1581 struct ww_acquire_ctx *ticket) 1614 1582 { 1615 - struct amdgpu_bo_va *bo_va, *tmp; 1583 + struct amdgpu_bo_va *bo_va; 1616 1584 struct dma_resv *resv; 1617 1585 bool clear, unlock; 1618 1586 int r; 1619 1587 1620 - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 1588 + spin_lock(&vm->status_lock); 1589 + while (!list_empty(&vm->moved)) { 1590 + bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, 1591 + base.vm_status); 1592 + spin_unlock(&vm->status_lock); 1593 + 1621 1594 /* Per VM BOs never need to bo cleared in the page tables */ 1622 1595 r = amdgpu_vm_bo_update(adev, bo_va, false); 1623 1596 if (r) 1624 1597 return r; 1598 + spin_lock(&vm->status_lock); 1625 1599 } 1626 1600 1627 - spin_lock(&vm->invalidated_lock); 1628 1601 while (!list_empty(&vm->invalidated)) { 1629 1602 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 1630 1603 base.vm_status); 1631 1604 resv = bo_va->base.bo->tbo.base.resv; 1632 - spin_unlock(&vm->invalidated_lock); 1605 + spin_unlock(&vm->status_lock); 1633 1606 1634 1607 /* Try to reserve the BO to avoid clearing its ptes */ 1635 1608 if (!adev->debug_vm && dma_resv_trylock(resv)) { ··· 1666 1629 bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM)) 1667 1630 amdgpu_vm_bo_evicted_user(&bo_va->base); 1668 1631 1669 - spin_lock(&vm->invalidated_lock); 1632 + spin_lock(&vm->status_lock); 1670 1633 } 1671 - spin_unlock(&vm->invalidated_lock); 1634 + spin_unlock(&vm->status_lock); 1672 1635 1673 1636 return 0; 1674 1637 } ··· 2211 2174 } 2212 2175 } 2213 2176 2214 - spin_lock(&vm->invalidated_lock); 2177 + spin_lock(&vm->status_lock); 2215 2178 list_del(&bo_va->base.vm_status); 2216 - spin_unlock(&vm->invalidated_lock); 2179 + spin_unlock(&vm->status_lock); 2217 2180 2218 2181 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2219 2182 list_del(&mapping->list); ··· 2321 2284 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2322 2285 struct amdgpu_vm *vm = bo_base->vm; 2323 2286 2324 - spin_lock(&vm->stats_lock); 2287 + spin_lock(&vm->status_lock); 2325 2288 amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1); 2326 2289 amdgpu_vm_update_stats_locked(bo_base, new_mem, +1); 2327 - spin_unlock(&vm->stats_lock); 2290 + spin_unlock(&vm->status_lock); 2328 2291 } 2329 2292 2330 2293 amdgpu_vm_bo_invalidate(bo, evicted); ··· 2593 2556 INIT_LIST_HEAD(&vm->relocated); 2594 2557 INIT_LIST_HEAD(&vm->moved); 2595 2558 INIT_LIST_HEAD(&vm->idle); 2596 - spin_lock_init(&vm->invalidated_lock); 2597 2559 INIT_LIST_HEAD(&vm->invalidated); 2560 + spin_lock_init(&vm->status_lock); 2598 2561 INIT_LIST_HEAD(&vm->freed); 2599 2562 INIT_LIST_HEAD(&vm->done); 2600 2563 INIT_KFIFO(vm->faults); 2601 - spin_lock_init(&vm->stats_lock); 2602 2564 2603 2565 r = amdgpu_vm_init_entities(adev, vm); 2604 2566 if (r) ··· 3065 3029 3066 3030 amdgpu_vm_assert_locked(vm); 3067 3031 3032 + spin_lock(&vm->status_lock); 3068 3033 seq_puts(m, "\tIdle BOs:\n"); 3069 3034 list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { 3070 3035 if (!bo_va->base.bo) ··· 3103 3066 id = 0; 3104 3067 3105 3068 seq_puts(m, "\tInvalidated BOs:\n"); 3106 - spin_lock(&vm->invalidated_lock); 3107 3069 list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { 3108 3070 if (!bo_va->base.bo) 3109 3071 continue; 3110 3072 total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3111 3073 } 3112 - spin_unlock(&vm->invalidated_lock); 3113 3074 total_invalidated_objs = id; 3114 3075 id = 0; 3115 3076 ··· 3117 3082 continue; 3118 3083 total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3119 3084 } 3085 + spin_unlock(&vm->status_lock); 3120 3086 total_done_objs = id; 3121 3087 3122 3088 seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,

+6 -9

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h

··· 205 205 /* protected by bo being reserved */ 206 206 struct amdgpu_vm_bo_base *next; 207 207 208 - /* protected by vm reservation and invalidated_lock */ 208 + /* protected by vm status_lock */ 209 209 struct list_head vm_status; 210 210 211 211 /* if the bo is counted as shared in mem stats 212 - * protected by vm BO being reserved */ 212 + * protected by vm status_lock */ 213 213 bool shared; 214 214 215 215 /* protected by the BO being reserved */ ··· 345 345 bool evicting; 346 346 unsigned int saved_flags; 347 347 348 - /* Memory statistics for this vm, protected by stats_lock */ 349 - spinlock_t stats_lock; 348 + /* Lock to protect vm_bo add/del/move on all lists of vm */ 349 + spinlock_t status_lock; 350 + 351 + /* Memory statistics for this vm, protected by status_lock */ 350 352 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]; 351 353 352 354 /* ··· 356 354 * PDs, PTs or per VM BOs. The state transits are: 357 355 * 358 356 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle 359 - * 360 - * Lists are protected by the root PD dma_resv lock. 361 357 */ 362 358 363 359 /* Per-VM and PT BOs who needs a validation */ ··· 376 376 * state transits are: 377 377 * 378 378 * evicted_user or invalidated -> done 379 - * 380 - * Lists are protected by the invalidated_lock. 381 379 */ 382 - spinlock_t invalidated_lock; 383 380 384 381 /* BOs for user mode queues that need a validation */ 385 382 struct list_head evicted_user;

+4

drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c

··· 544 544 entry->bo->vm_bo = NULL; 545 545 ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); 546 546 547 + spin_lock(&entry->vm->status_lock); 547 548 list_del(&entry->vm_status); 549 + spin_unlock(&entry->vm->status_lock); 548 550 amdgpu_bo_unref(&entry->bo); 549 551 } 550 552 ··· 590 588 struct amdgpu_vm_pt_cursor seek; 591 589 struct amdgpu_vm_bo_base *entry; 592 590 591 + spin_lock(&params->vm->status_lock); 593 592 for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) { 594 593 if (entry && entry->bo) 595 594 list_move(&entry->vm_status, &params->tlb_flush_waitlist); ··· 598 595 599 596 /* enter start node now */ 600 597 list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist); 598 + spin_unlock(&params->vm->status_lock); 601 599 } 602 600 603 601 /**

Configure Feed

Configure Feed