drm/amdgpu: separate VM PT handling into amdgpu_vm_pt.c

+1 -1

drivers/gpu/drm/amd/amdgpu/Makefile

··· 49 49 amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o \ 50 50 atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \ 51 51 atombios_encoders.o amdgpu_sa.o atombios_i2c.o \ 52 - amdgpu_dma_buf.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \ 52 + amdgpu_dma_buf.o amdgpu_vm.o amdgpu_vm_pt.o amdgpu_ib.o amdgpu_pll.o \ 53 53 amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \ 54 54 amdgpu_gtt_mgr.o amdgpu_preempt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o \ 55 55 amdgpu_atomfirmware.o amdgpu_vf_error.o amdgpu_sched.o \

+10 -943

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

··· 155 155 } 156 156 157 157 /** 158 - * amdgpu_vm_level_shift - return the addr shift for each level 159 - * 160 - * @adev: amdgpu_device pointer 161 - * @level: VMPT level 162 - * 163 - * Returns: 164 - * The number of bits the pfn needs to be right shifted for a level. 165 - */ 166 - static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, 167 - unsigned level) 168 - { 169 - switch (level) { 170 - case AMDGPU_VM_PDB2: 171 - case AMDGPU_VM_PDB1: 172 - case AMDGPU_VM_PDB0: 173 - return 9 * (AMDGPU_VM_PDB0 - level) + 174 - adev->vm_manager.block_size; 175 - case AMDGPU_VM_PTB: 176 - return 0; 177 - default: 178 - return ~0; 179 - } 180 - } 181 - 182 - /** 183 - * amdgpu_vm_num_entries - return the number of entries in a PD/PT 184 - * 185 - * @adev: amdgpu_device pointer 186 - * @level: VMPT level 187 - * 188 - * Returns: 189 - * The number of entries in a page directory or page table. 190 - */ 191 - static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 192 - unsigned level) 193 - { 194 - unsigned shift = amdgpu_vm_level_shift(adev, 195 - adev->vm_manager.root_level); 196 - 197 - if (level == adev->vm_manager.root_level) 198 - /* For the root directory */ 199 - return round_up(adev->vm_manager.max_pfn, 1ULL << shift) 200 - >> shift; 201 - else if (level != AMDGPU_VM_PTB) 202 - /* Everything in between */ 203 - return 512; 204 - else 205 - /* For the page tables on the leaves */ 206 - return AMDGPU_VM_PTE_COUNT(adev); 207 - } 208 - 209 - /** 210 - * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD 211 - * 212 - * @adev: amdgpu_device pointer 213 - * 214 - * Returns: 215 - * The number of entries in the root page directory which needs the ATS setting. 216 - */ 217 - static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev) 218 - { 219 - unsigned shift; 220 - 221 - shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level); 222 - return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); 223 - } 224 - 225 - /** 226 - * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT 227 - * 228 - * @adev: amdgpu_device pointer 229 - * @level: VMPT level 230 - * 231 - * Returns: 232 - * The mask to extract the entry number of a PD/PT from an address. 233 - */ 234 - static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev, 235 - unsigned int level) 236 - { 237 - if (level <= adev->vm_manager.root_level) 238 - return 0xffffffff; 239 - else if (level != AMDGPU_VM_PTB) 240 - return 0x1ff; 241 - else 242 - return AMDGPU_VM_PTE_COUNT(adev) - 1; 243 - } 244 - 245 - /** 246 - * amdgpu_vm_bo_size - returns the size of the BOs in bytes 247 - * 248 - * @adev: amdgpu_device pointer 249 - * @level: VMPT level 250 - * 251 - * Returns: 252 - * The size of the BO for a page directory or page table in bytes. 253 - */ 254 - static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) 255 - { 256 - return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); 257 - } 258 - 259 - /** 260 158 * amdgpu_vm_bo_evicted - vm_bo is evicted 261 159 * 262 160 * @vm_bo: vm_bo which is evicted ··· 256 358 * Initialize a bo_va_base structure and add it to the appropriate lists 257 359 * 258 360 */ 259 - static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 260 - struct amdgpu_vm *vm, 261 - struct amdgpu_bo *bo) 361 + void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 362 + struct amdgpu_vm *vm, struct amdgpu_bo *bo) 262 363 { 263 364 base->vm = vm; 264 365 base->bo = bo; ··· 291 394 * */ 292 395 amdgpu_vm_bo_evicted(base); 293 396 } 294 - 295 - /** 296 - * amdgpu_vm_pt_parent - get the parent page directory 297 - * 298 - * @pt: child page table 299 - * 300 - * Helper to get the parent entry for the child page table. NULL if we are at 301 - * the root page directory. 302 - */ 303 - static struct amdgpu_vm_bo_base *amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt) 304 - { 305 - struct amdgpu_bo *parent = pt->bo->parent; 306 - 307 - if (!parent) 308 - return NULL; 309 - 310 - return parent->vm_bo; 311 - } 312 - 313 - /* 314 - * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 315 - */ 316 - struct amdgpu_vm_pt_cursor { 317 - uint64_t pfn; 318 - struct amdgpu_vm_bo_base *parent; 319 - struct amdgpu_vm_bo_base *entry; 320 - unsigned level; 321 - }; 322 - 323 - /** 324 - * amdgpu_vm_pt_start - start PD/PT walk 325 - * 326 - * @adev: amdgpu_device pointer 327 - * @vm: amdgpu_vm structure 328 - * @start: start address of the walk 329 - * @cursor: state to initialize 330 - * 331 - * Initialize a amdgpu_vm_pt_cursor to start a walk. 332 - */ 333 - static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 334 - struct amdgpu_vm *vm, uint64_t start, 335 - struct amdgpu_vm_pt_cursor *cursor) 336 - { 337 - cursor->pfn = start; 338 - cursor->parent = NULL; 339 - cursor->entry = &vm->root; 340 - cursor->level = adev->vm_manager.root_level; 341 - } 342 - 343 - /** 344 - * amdgpu_vm_pt_descendant - go to child node 345 - * 346 - * @adev: amdgpu_device pointer 347 - * @cursor: current state 348 - * 349 - * Walk to the child node of the current node. 350 - * Returns: 351 - * True if the walk was possible, false otherwise. 352 - */ 353 - static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 354 - struct amdgpu_vm_pt_cursor *cursor) 355 - { 356 - unsigned mask, shift, idx; 357 - 358 - if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || 359 - !cursor->entry->bo) 360 - return false; 361 - 362 - mask = amdgpu_vm_entries_mask(adev, cursor->level); 363 - shift = amdgpu_vm_level_shift(adev, cursor->level); 364 - 365 - ++cursor->level; 366 - idx = (cursor->pfn >> shift) & mask; 367 - cursor->parent = cursor->entry; 368 - cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; 369 - return true; 370 - } 371 - 372 - /** 373 - * amdgpu_vm_pt_sibling - go to sibling node 374 - * 375 - * @adev: amdgpu_device pointer 376 - * @cursor: current state 377 - * 378 - * Walk to the sibling node of the current node. 379 - * Returns: 380 - * True if the walk was possible, false otherwise. 381 - */ 382 - static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 383 - struct amdgpu_vm_pt_cursor *cursor) 384 - { 385 - unsigned shift, num_entries; 386 - 387 - /* Root doesn't have a sibling */ 388 - if (!cursor->parent) 389 - return false; 390 - 391 - /* Go to our parents and see if we got a sibling */ 392 - shift = amdgpu_vm_level_shift(adev, cursor->level - 1); 393 - num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1); 394 - 395 - if (cursor->entry == &to_amdgpu_bo_vm(cursor->parent->bo)->entries[num_entries - 1]) 396 - return false; 397 - 398 - cursor->pfn += 1ULL << shift; 399 - cursor->pfn &= ~((1ULL << shift) - 1); 400 - ++cursor->entry; 401 - return true; 402 - } 403 - 404 - /** 405 - * amdgpu_vm_pt_ancestor - go to parent node 406 - * 407 - * @cursor: current state 408 - * 409 - * Walk to the parent node of the current node. 410 - * Returns: 411 - * True if the walk was possible, false otherwise. 412 - */ 413 - static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 414 - { 415 - if (!cursor->parent) 416 - return false; 417 - 418 - --cursor->level; 419 - cursor->entry = cursor->parent; 420 - cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 421 - return true; 422 - } 423 - 424 - /** 425 - * amdgpu_vm_pt_next - get next PD/PT in hieratchy 426 - * 427 - * @adev: amdgpu_device pointer 428 - * @cursor: current state 429 - * 430 - * Walk the PD/PT tree to the next node. 431 - */ 432 - static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 433 - struct amdgpu_vm_pt_cursor *cursor) 434 - { 435 - /* First try a newborn child */ 436 - if (amdgpu_vm_pt_descendant(adev, cursor)) 437 - return; 438 - 439 - /* If that didn't worked try to find a sibling */ 440 - while (!amdgpu_vm_pt_sibling(adev, cursor)) { 441 - /* No sibling, go to our parents and grandparents */ 442 - if (!amdgpu_vm_pt_ancestor(cursor)) { 443 - cursor->pfn = ~0ll; 444 - return; 445 - } 446 - } 447 - } 448 - 449 - /** 450 - * amdgpu_vm_pt_first_dfs - start a deep first search 451 - * 452 - * @adev: amdgpu_device structure 453 - * @vm: amdgpu_vm structure 454 - * @start: optional cursor to start with 455 - * @cursor: state to initialize 456 - * 457 - * Starts a deep first traversal of the PD/PT tree. 458 - */ 459 - static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 460 - struct amdgpu_vm *vm, 461 - struct amdgpu_vm_pt_cursor *start, 462 - struct amdgpu_vm_pt_cursor *cursor) 463 - { 464 - if (start) 465 - *cursor = *start; 466 - else 467 - amdgpu_vm_pt_start(adev, vm, 0, cursor); 468 - while (amdgpu_vm_pt_descendant(adev, cursor)); 469 - } 470 - 471 - /** 472 - * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 473 - * 474 - * @start: starting point for the search 475 - * @entry: current entry 476 - * 477 - * Returns: 478 - * True when the search should continue, false otherwise. 479 - */ 480 - static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 481 - struct amdgpu_vm_bo_base *entry) 482 - { 483 - return entry && (!start || entry != start->entry); 484 - } 485 - 486 - /** 487 - * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 488 - * 489 - * @adev: amdgpu_device structure 490 - * @cursor: current state 491 - * 492 - * Move the cursor to the next node in a deep first search. 493 - */ 494 - static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 495 - struct amdgpu_vm_pt_cursor *cursor) 496 - { 497 - if (!cursor->entry) 498 - return; 499 - 500 - if (!cursor->parent) 501 - cursor->entry = NULL; 502 - else if (amdgpu_vm_pt_sibling(adev, cursor)) 503 - while (amdgpu_vm_pt_descendant(adev, cursor)); 504 - else 505 - amdgpu_vm_pt_ancestor(cursor); 506 - } 507 - 508 - /* 509 - * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 510 - */ 511 - #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 512 - for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 513 - (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 514 - amdgpu_vm_pt_continue_dfs((start), (entry)); \ 515 - (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 516 397 517 398 /** 518 399 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list ··· 456 781 amdgpu_vm_eviction_unlock(vm); 457 782 458 783 return ret && list_empty(&vm->evicted); 459 - } 460 - 461 - /** 462 - * amdgpu_vm_clear_bo - initially clear the PDs/PTs 463 - * 464 - * @adev: amdgpu_device pointer 465 - * @vm: VM to clear BO from 466 - * @vmbo: BO to clear 467 - * @immediate: use an immediate update 468 - * 469 - * Root PD needs to be reserved when calling this. 470 - * 471 - * Returns: 472 - * 0 on success, errno otherwise. 473 - */ 474 - static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 475 - struct amdgpu_vm *vm, 476 - struct amdgpu_bo_vm *vmbo, 477 - bool immediate) 478 - { 479 - struct ttm_operation_ctx ctx = { true, false }; 480 - unsigned level = adev->vm_manager.root_level; 481 - struct amdgpu_vm_update_params params; 482 - struct amdgpu_bo *ancestor = &vmbo->bo; 483 - struct amdgpu_bo *bo = &vmbo->bo; 484 - unsigned entries, ats_entries; 485 - uint64_t addr; 486 - int r, idx; 487 - 488 - /* Figure out our place in the hierarchy */ 489 - if (ancestor->parent) { 490 - ++level; 491 - while (ancestor->parent->parent) { 492 - ++level; 493 - ancestor = ancestor->parent; 494 - } 495 - } 496 - 497 - entries = amdgpu_bo_size(bo) / 8; 498 - if (!vm->pte_support_ats) { 499 - ats_entries = 0; 500 - 501 - } else if (!bo->parent) { 502 - ats_entries = amdgpu_vm_num_ats_entries(adev); 503 - ats_entries = min(ats_entries, entries); 504 - entries -= ats_entries; 505 - 506 - } else { 507 - struct amdgpu_vm_bo_base *pt; 508 - 509 - pt = ancestor->vm_bo; 510 - ats_entries = amdgpu_vm_num_ats_entries(adev); 511 - if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >= ats_entries) { 512 - ats_entries = 0; 513 - } else { 514 - ats_entries = entries; 515 - entries = 0; 516 - } 517 - } 518 - 519 - r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 520 - if (r) 521 - return r; 522 - 523 - if (vmbo->shadow) { 524 - struct amdgpu_bo *shadow = vmbo->shadow; 525 - 526 - r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx); 527 - if (r) 528 - return r; 529 - } 530 - 531 - if (!drm_dev_enter(adev_to_drm(adev), &idx)) 532 - return -ENODEV; 533 - 534 - r = vm->update_funcs->map_table(vmbo); 535 - if (r) 536 - goto exit; 537 - 538 - memset(&params, 0, sizeof(params)); 539 - params.adev = adev; 540 - params.vm = vm; 541 - params.immediate = immediate; 542 - 543 - r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT); 544 - if (r) 545 - goto exit; 546 - 547 - addr = 0; 548 - if (ats_entries) { 549 - uint64_t value = 0, flags; 550 - 551 - flags = AMDGPU_PTE_DEFAULT_ATC; 552 - if (level != AMDGPU_VM_PTB) { 553 - /* Handle leaf PDEs as PTEs */ 554 - flags |= AMDGPU_PDE_PTE; 555 - amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); 556 - } 557 - 558 - r = vm->update_funcs->update(&params, vmbo, addr, 0, ats_entries, 559 - value, flags); 560 - if (r) 561 - goto exit; 562 - 563 - addr += ats_entries * 8; 564 - } 565 - 566 - if (entries) { 567 - uint64_t value = 0, flags = 0; 568 - 569 - if (adev->asic_type >= CHIP_VEGA10) { 570 - if (level != AMDGPU_VM_PTB) { 571 - /* Handle leaf PDEs as PTEs */ 572 - flags |= AMDGPU_PDE_PTE; 573 - amdgpu_gmc_get_vm_pde(adev, level, 574 - &value, &flags); 575 - } else { 576 - /* Workaround for fault priority problem on GMC9 */ 577 - flags = AMDGPU_PTE_EXECUTABLE; 578 - } 579 - } 580 - 581 - r = vm->update_funcs->update(&params, vmbo, addr, 0, entries, 582 - value, flags); 583 - if (r) 584 - goto exit; 585 - } 586 - 587 - r = vm->update_funcs->commit(&params, NULL); 588 - exit: 589 - drm_dev_exit(idx); 590 - return r; 591 - } 592 - 593 - /** 594 - * amdgpu_vm_pt_create - create bo for PD/PT 595 - * 596 - * @adev: amdgpu_device pointer 597 - * @vm: requesting vm 598 - * @level: the page table level 599 - * @immediate: use a immediate update 600 - * @vmbo: pointer to the buffer object pointer 601 - */ 602 - static int amdgpu_vm_pt_create(struct amdgpu_device *adev, 603 - struct amdgpu_vm *vm, 604 - int level, bool immediate, 605 - struct amdgpu_bo_vm **vmbo) 606 - { 607 - struct amdgpu_bo_param bp; 608 - struct amdgpu_bo *bo; 609 - struct dma_resv *resv; 610 - unsigned int num_entries; 611 - int r; 612 - 613 - memset(&bp, 0, sizeof(bp)); 614 - 615 - bp.size = amdgpu_vm_bo_size(adev, level); 616 - bp.byte_align = AMDGPU_GPU_PAGE_SIZE; 617 - bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 618 - bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); 619 - bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 620 - AMDGPU_GEM_CREATE_CPU_GTT_USWC; 621 - 622 - if (level < AMDGPU_VM_PTB) 623 - num_entries = amdgpu_vm_num_entries(adev, level); 624 - else 625 - num_entries = 0; 626 - 627 - bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries); 628 - 629 - if (vm->use_cpu_for_update) 630 - bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 631 - 632 - bp.type = ttm_bo_type_kernel; 633 - bp.no_wait_gpu = immediate; 634 - if (vm->root.bo) 635 - bp.resv = vm->root.bo->tbo.base.resv; 636 - 637 - r = amdgpu_bo_create_vm(adev, &bp, vmbo); 638 - if (r) 639 - return r; 640 - 641 - bo = &(*vmbo)->bo; 642 - if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) { 643 - (*vmbo)->shadow = NULL; 644 - return 0; 645 - } 646 - 647 - if (!bp.resv) 648 - WARN_ON(dma_resv_lock(bo->tbo.base.resv, 649 - NULL)); 650 - resv = bp.resv; 651 - memset(&bp, 0, sizeof(bp)); 652 - bp.size = amdgpu_vm_bo_size(adev, level); 653 - bp.domain = AMDGPU_GEM_DOMAIN_GTT; 654 - bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; 655 - bp.type = ttm_bo_type_kernel; 656 - bp.resv = bo->tbo.base.resv; 657 - bp.bo_ptr_size = sizeof(struct amdgpu_bo); 658 - 659 - r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow); 660 - 661 - if (!resv) 662 - dma_resv_unlock(bo->tbo.base.resv); 663 - 664 - if (r) { 665 - amdgpu_bo_unref(&bo); 666 - return r; 667 - } 668 - 669 - (*vmbo)->shadow->parent = amdgpu_bo_ref(bo); 670 - amdgpu_bo_add_to_shadow_list(*vmbo); 671 - 672 - return 0; 673 - } 674 - 675 - /** 676 - * amdgpu_vm_alloc_pts - Allocate a specific page table 677 - * 678 - * @adev: amdgpu_device pointer 679 - * @vm: VM to allocate page tables for 680 - * @cursor: Which page table to allocate 681 - * @immediate: use an immediate update 682 - * 683 - * Make sure a specific page table or directory is allocated. 684 - * 685 - * Returns: 686 - * 1 if page table needed to be allocated, 0 if page table was already 687 - * allocated, negative errno if an error occurred. 688 - */ 689 - static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, 690 - struct amdgpu_vm *vm, 691 - struct amdgpu_vm_pt_cursor *cursor, 692 - bool immediate) 693 - { 694 - struct amdgpu_vm_bo_base *entry = cursor->entry; 695 - struct amdgpu_bo *pt_bo; 696 - struct amdgpu_bo_vm *pt; 697 - int r; 698 - 699 - if (entry->bo) 700 - return 0; 701 - 702 - r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt); 703 - if (r) 704 - return r; 705 - 706 - /* Keep a reference to the root directory to avoid 707 - * freeing them up in the wrong order. 708 - */ 709 - pt_bo = &pt->bo; 710 - pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); 711 - amdgpu_vm_bo_base_init(entry, vm, pt_bo); 712 - r = amdgpu_vm_clear_bo(adev, vm, pt, immediate); 713 - if (r) 714 - goto error_free_pt; 715 - 716 - return 0; 717 - 718 - error_free_pt: 719 - amdgpu_bo_unref(&pt->shadow); 720 - amdgpu_bo_unref(&pt_bo); 721 - return r; 722 - } 723 - 724 - /** 725 - * amdgpu_vm_free_table - fre one PD/PT 726 - * 727 - * @entry: PDE to free 728 - */ 729 - static void amdgpu_vm_free_table(struct amdgpu_vm_bo_base *entry) 730 - { 731 - struct amdgpu_bo *shadow; 732 - 733 - if (!entry->bo) 734 - return; 735 - shadow = amdgpu_bo_shadowed(entry->bo); 736 - entry->bo->vm_bo = NULL; 737 - list_del(&entry->vm_status); 738 - amdgpu_bo_unref(&shadow); 739 - amdgpu_bo_unref(&entry->bo); 740 - } 741 - 742 - /** 743 - * amdgpu_vm_free_pts - free PD/PT levels 744 - * 745 - * @adev: amdgpu device structure 746 - * @vm: amdgpu vm structure 747 - * @start: optional cursor where to start freeing PDs/PTs 748 - * 749 - * Free the page directory or page table level and all sub levels. 750 - */ 751 - static void amdgpu_vm_free_pts(struct amdgpu_device *adev, 752 - struct amdgpu_vm *vm, 753 - struct amdgpu_vm_pt_cursor *start) 754 - { 755 - struct amdgpu_vm_pt_cursor cursor; 756 - struct amdgpu_vm_bo_base *entry; 757 - 758 - vm->bulk_moveable = false; 759 - 760 - for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) 761 - amdgpu_vm_free_table(entry); 762 - 763 - if (start) 764 - amdgpu_vm_free_table(start->entry); 765 784 } 766 785 767 786 /** ··· 705 1336 } 706 1337 707 1338 /** 708 - * amdgpu_vm_update_pde - update a single level in the hierarchy 709 - * 710 - * @params: parameters for the update 711 - * @vm: requested vm 712 - * @entry: entry to update 713 - * 714 - * Makes sure the requested entry in parent is up to date. 715 - */ 716 - static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params, 717 - struct amdgpu_vm *vm, 718 - struct amdgpu_vm_bo_base *entry) 719 - { 720 - struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry); 721 - struct amdgpu_bo *bo = parent->bo, *pbo; 722 - uint64_t pde, pt, flags; 723 - unsigned level; 724 - 725 - for (level = 0, pbo = bo->parent; pbo; ++level) 726 - pbo = pbo->parent; 727 - 728 - level += params->adev->vm_manager.root_level; 729 - amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags); 730 - pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8; 731 - return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt, 732 - 1, 0, flags); 733 - } 734 - 735 - /** 736 1339 * amdgpu_vm_update_pdes - make sure that all directories are valid 737 1340 * 738 1341 * @adev: amdgpu_device pointer ··· 739 1398 goto error; 740 1399 741 1400 list_for_each_entry(entry, &vm->relocated, vm_status) { 742 - r = amdgpu_vm_update_pde(&params, vm, entry); 1401 + r = amdgpu_vm_pde_update(&params, entry); 743 1402 if (r) 744 1403 goto error; 745 1404 } ··· 758 1417 error: 759 1418 drm_dev_exit(idx); 760 1419 return r; 761 - } 762 - 763 - /* 764 - * amdgpu_vm_update_flags - figure out flags for PTE updates 765 - * 766 - * Make sure to set the right flags for the PTEs at the desired level. 767 - */ 768 - static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params, 769 - struct amdgpu_bo_vm *pt, unsigned int level, 770 - uint64_t pe, uint64_t addr, 771 - unsigned int count, uint32_t incr, 772 - uint64_t flags) 773 - 774 - { 775 - if (level != AMDGPU_VM_PTB) { 776 - flags |= AMDGPU_PDE_PTE; 777 - amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); 778 - 779 - } else if (params->adev->asic_type >= CHIP_VEGA10 && 780 - !(flags & AMDGPU_PTE_VALID) && 781 - !(flags & AMDGPU_PTE_PRT)) { 782 - 783 - /* Workaround for fault priority problem on GMC9 */ 784 - flags |= AMDGPU_PTE_EXECUTABLE; 785 - } 786 - 787 - params->vm->update_funcs->update(params, pt, pe, addr, count, incr, 788 - flags); 789 - } 790 - 791 - /** 792 - * amdgpu_vm_fragment - get fragment for PTEs 793 - * 794 - * @params: see amdgpu_vm_update_params definition 795 - * @start: first PTE to handle 796 - * @end: last PTE to handle 797 - * @flags: hw mapping flags 798 - * @frag: resulting fragment size 799 - * @frag_end: end of this fragment 800 - * 801 - * Returns the first possible fragment for the start and end address. 802 - */ 803 - static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params, 804 - uint64_t start, uint64_t end, uint64_t flags, 805 - unsigned int *frag, uint64_t *frag_end) 806 - { 807 - /** 808 - * The MC L1 TLB supports variable sized pages, based on a fragment 809 - * field in the PTE. When this field is set to a non-zero value, page 810 - * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 811 - * flags are considered valid for all PTEs within the fragment range 812 - * and corresponding mappings are assumed to be physically contiguous. 813 - * 814 - * The L1 TLB can store a single PTE for the whole fragment, 815 - * significantly increasing the space available for translation 816 - * caching. This leads to large improvements in throughput when the 817 - * TLB is under pressure. 818 - * 819 - * The L2 TLB distributes small and large fragments into two 820 - * asymmetric partitions. The large fragment cache is significantly 821 - * larger. Thus, we try to use large fragments wherever possible. 822 - * Userspace can support this by aligning virtual base address and 823 - * allocation size to the fragment size. 824 - * 825 - * Starting with Vega10 the fragment size only controls the L1. The L2 826 - * is now directly feed with small/huge/giant pages from the walker. 827 - */ 828 - unsigned max_frag; 829 - 830 - if (params->adev->asic_type < CHIP_VEGA10) 831 - max_frag = params->adev->vm_manager.fragment_size; 832 - else 833 - max_frag = 31; 834 - 835 - /* system pages are non continuously */ 836 - if (params->pages_addr) { 837 - *frag = 0; 838 - *frag_end = end; 839 - return; 840 - } 841 - 842 - /* This intentionally wraps around if no bit is set */ 843 - *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1); 844 - if (*frag >= max_frag) { 845 - *frag = max_frag; 846 - *frag_end = end & ~((1ULL << max_frag) - 1); 847 - } else { 848 - *frag_end = start + (1 << *frag); 849 - } 850 - } 851 - 852 - /** 853 - * amdgpu_vm_update_ptes - make sure that page tables are valid 854 - * 855 - * @params: see amdgpu_vm_update_params definition 856 - * @start: start of GPU address range 857 - * @end: end of GPU address range 858 - * @dst: destination address to map to, the next dst inside the function 859 - * @flags: mapping flags 860 - * 861 - * Update the page tables in the range @start - @end. 862 - * 863 - * Returns: 864 - * 0 for success, -EINVAL for failure. 865 - */ 866 - static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, 867 - uint64_t start, uint64_t end, 868 - uint64_t dst, uint64_t flags) 869 - { 870 - struct amdgpu_device *adev = params->adev; 871 - struct amdgpu_vm_pt_cursor cursor; 872 - uint64_t frag_start = start, frag_end; 873 - unsigned int frag; 874 - int r; 875 - 876 - /* figure out the initial fragment */ 877 - amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end); 878 - 879 - /* walk over the address space and update the PTs */ 880 - amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 881 - while (cursor.pfn < end) { 882 - unsigned shift, parent_shift, mask; 883 - uint64_t incr, entry_end, pe_start; 884 - struct amdgpu_bo *pt; 885 - 886 - if (!params->unlocked) { 887 - /* make sure that the page tables covering the 888 - * address range are actually allocated 889 - */ 890 - r = amdgpu_vm_alloc_pts(params->adev, params->vm, 891 - &cursor, params->immediate); 892 - if (r) 893 - return r; 894 - } 895 - 896 - shift = amdgpu_vm_level_shift(adev, cursor.level); 897 - parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1); 898 - if (params->unlocked) { 899 - /* Unlocked updates are only allowed on the leaves */ 900 - if (amdgpu_vm_pt_descendant(adev, &cursor)) 901 - continue; 902 - } else if (adev->asic_type < CHIP_VEGA10 && 903 - (flags & AMDGPU_PTE_VALID)) { 904 - /* No huge page support before GMC v9 */ 905 - if (cursor.level != AMDGPU_VM_PTB) { 906 - if (!amdgpu_vm_pt_descendant(adev, &cursor)) 907 - return -ENOENT; 908 - continue; 909 - } 910 - } else if (frag < shift) { 911 - /* We can't use this level when the fragment size is 912 - * smaller than the address shift. Go to the next 913 - * child entry and try again. 914 - */ 915 - if (amdgpu_vm_pt_descendant(adev, &cursor)) 916 - continue; 917 - } else if (frag >= parent_shift) { 918 - /* If the fragment size is even larger than the parent 919 - * shift we should go up one level and check it again. 920 - */ 921 - if (!amdgpu_vm_pt_ancestor(&cursor)) 922 - return -EINVAL; 923 - continue; 924 - } 925 - 926 - pt = cursor.entry->bo; 927 - if (!pt) { 928 - /* We need all PDs and PTs for mapping something, */ 929 - if (flags & AMDGPU_PTE_VALID) 930 - return -ENOENT; 931 - 932 - /* but unmapping something can happen at a higher 933 - * level. 934 - */ 935 - if (!amdgpu_vm_pt_ancestor(&cursor)) 936 - return -EINVAL; 937 - 938 - pt = cursor.entry->bo; 939 - shift = parent_shift; 940 - frag_end = max(frag_end, ALIGN(frag_start + 1, 941 - 1ULL << shift)); 942 - } 943 - 944 - /* Looks good so far, calculate parameters for the update */ 945 - incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 946 - mask = amdgpu_vm_entries_mask(adev, cursor.level); 947 - pe_start = ((cursor.pfn >> shift) & mask) * 8; 948 - entry_end = ((uint64_t)mask + 1) << shift; 949 - entry_end += cursor.pfn & ~(entry_end - 1); 950 - entry_end = min(entry_end, end); 951 - 952 - do { 953 - struct amdgpu_vm *vm = params->vm; 954 - uint64_t upd_end = min(entry_end, frag_end); 955 - unsigned nptes = (upd_end - frag_start) >> shift; 956 - uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag); 957 - 958 - /* This can happen when we set higher level PDs to 959 - * silent to stop fault floods. 960 - */ 961 - nptes = max(nptes, 1u); 962 - 963 - trace_amdgpu_vm_update_ptes(params, frag_start, upd_end, 964 - min(nptes, 32u), dst, incr, upd_flags, 965 - vm->task_info.pid, 966 - vm->immediate.fence_context); 967 - amdgpu_vm_update_flags(params, to_amdgpu_bo_vm(pt), 968 - cursor.level, pe_start, dst, 969 - nptes, incr, upd_flags); 970 - 971 - pe_start += nptes * 8; 972 - dst += nptes * incr; 973 - 974 - frag_start = upd_end; 975 - if (frag_start >= frag_end) { 976 - /* figure out the next fragment */ 977 - amdgpu_vm_fragment(params, frag_start, end, 978 - flags, &frag, &frag_end); 979 - if (frag < shift) 980 - break; 981 - } 982 - } while (frag_start < entry_end); 983 - 984 - if (amdgpu_vm_pt_descendant(adev, &cursor)) { 985 - /* Free all child entries. 986 - * Update the tables with the flags and addresses and free up subsequent 987 - * tables in the case of huge pages or freed up areas. 988 - * This is the maximum you can free, because all other page tables are not 989 - * completely covered by the range and so potentially still in use. 990 - */ 991 - while (cursor.pfn < frag_start) { 992 - /* Make sure previous mapping is freed */ 993 - if (cursor.entry->bo) { 994 - params->table_freed = true; 995 - amdgpu_vm_free_pts(adev, params->vm, &cursor); 996 - } 997 - amdgpu_vm_pt_next(adev, &cursor); 998 - } 999 - 1000 - } else if (frag >= shift) { 1001 - /* or just move on to the next on the same level. */ 1002 - amdgpu_vm_pt_next(adev, &cursor); 1003 - } 1004 - } 1005 - 1006 - return 0; 1007 1420 } 1008 1421 1009 1422 /** ··· 880 1785 } 881 1786 882 1787 tmp = start + num_entries; 883 - r = amdgpu_vm_update_ptes(&params, start, tmp, addr, flags); 1788 + r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags); 884 1789 if (r) 885 1790 goto error_unlock; 886 1791 ··· 2059 2964 2060 2965 amdgpu_vm_bo_base_init(&vm->root, vm, root_bo); 2061 2966 2062 - r = amdgpu_vm_clear_bo(adev, vm, root, false); 2967 + r = amdgpu_vm_pt_clear(adev, vm, root, false); 2063 2968 if (r) 2064 2969 goto error_unreserve; 2065 2970 ··· 2085 2990 drm_sched_entity_destroy(&vm->immediate); 2086 2991 2087 2992 return r; 2088 - } 2089 - 2090 - /** 2091 - * amdgpu_vm_check_clean_reserved - check if a VM is clean 2092 - * 2093 - * @adev: amdgpu_device pointer 2094 - * @vm: the VM to check 2095 - * 2096 - * check all entries of the root PD, if any subsequent PDs are allocated, 2097 - * it means there are page table creating and filling, and is no a clean 2098 - * VM 2099 - * 2100 - * Returns: 2101 - * 0 if this VM is clean 2102 - */ 2103 - static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev, 2104 - struct amdgpu_vm *vm) 2105 - { 2106 - enum amdgpu_vm_level root = adev->vm_manager.root_level; 2107 - unsigned int entries = amdgpu_vm_num_entries(adev, root); 2108 - unsigned int i = 0; 2109 - 2110 - for (i = 0; i < entries; i++) { 2111 - if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo) 2112 - return -EINVAL; 2113 - } 2114 - 2115 - return 0; 2116 2993 } 2117 2994 2118 2995 /** ··· 2116 3049 return r; 2117 3050 2118 3051 /* Sanity checks */ 2119 - r = amdgpu_vm_check_clean_reserved(adev, vm); 2120 - if (r) 3052 + if (!amdgpu_vm_pt_is_root_clean(adev, vm)) { 3053 + r = -EINVAL; 2121 3054 goto unreserve_bo; 3055 + } 2122 3056 2123 3057 /* Check if PD needs to be reinitialized and do it before 2124 3058 * changing any other state, in case it fails. 2125 3059 */ 2126 3060 if (pte_support_ats != vm->pte_support_ats) { 2127 3061 vm->pte_support_ats = pte_support_ats; 2128 - r = amdgpu_vm_clear_bo(adev, vm, 2129 - to_amdgpu_bo_vm(vm->root.bo), 3062 + r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo), 2130 3063 false); 2131 3064 if (r) 2132 3065 goto unreserve_bo; ··· 2214 3147 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2215 3148 } 2216 3149 2217 - amdgpu_vm_free_pts(adev, vm, NULL); 3150 + amdgpu_vm_pt_free_root(adev, vm); 2218 3151 amdgpu_bo_unreserve(root); 2219 3152 amdgpu_bo_unref(&root); 2220 3153 WARN_ON(vm->root.bo);

+16

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h

··· 397 397 struct dma_fence **fence); 398 398 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 399 399 struct amdgpu_vm *vm); 400 + void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 401 + struct amdgpu_vm *vm, struct amdgpu_bo *bo); 400 402 int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 401 403 struct amdgpu_device *bo_adev, 402 404 struct amdgpu_vm *vm, bool immediate, ··· 459 457 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo); 460 458 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, 461 459 uint64_t *gtt_mem, uint64_t *cpu_mem); 460 + 461 + int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, 462 + struct amdgpu_bo_vm *vmbo, bool immediate); 463 + int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, 464 + int level, bool immediate, struct amdgpu_bo_vm **vmbo); 465 + void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm); 466 + bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev, 467 + struct amdgpu_vm *vm); 468 + 469 + int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 470 + struct amdgpu_vm_bo_base *entry); 471 + int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, 472 + uint64_t start, uint64_t end, 473 + uint64_t dst, uint64_t flags); 462 474 463 475 #if defined(CONFIG_DEBUG_FS) 464 476 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);

+979

drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 OR MIT 2 + /* 3 + * Copyright 2022 Advanced Micro Devices, Inc. 4 + * 5 + * Permission is hereby granted, free of charge, to any person obtaining a 6 + * copy of this software and associated documentation files (the "Software"), 7 + * to deal in the Software without restriction, including without limitation 8 + * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 + * and/or sell copies of the Software, and to permit persons to whom the 10 + * Software is furnished to do so, subject to the following conditions: 11 + * 12 + * The above copyright notice and this permission notice shall be included in 13 + * all copies or substantial portions of the Software. 14 + * 15 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 + * OTHER DEALINGS IN THE SOFTWARE. 22 + */ 23 + 24 + #include <drm/drm_drv.h> 25 + 26 + #include "amdgpu.h" 27 + #include "amdgpu_trace.h" 28 + #include "amdgpu_vm.h" 29 + 30 + /* 31 + * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 32 + */ 33 + struct amdgpu_vm_pt_cursor { 34 + uint64_t pfn; 35 + struct amdgpu_vm_bo_base *parent; 36 + struct amdgpu_vm_bo_base *entry; 37 + unsigned int level; 38 + }; 39 + 40 + /** 41 + * amdgpu_vm_pt_level_shift - return the addr shift for each level 42 + * 43 + * @adev: amdgpu_device pointer 44 + * @level: VMPT level 45 + * 46 + * Returns: 47 + * The number of bits the pfn needs to be right shifted for a level. 48 + */ 49 + static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, 50 + unsigned int level) 51 + { 52 + switch (level) { 53 + case AMDGPU_VM_PDB2: 54 + case AMDGPU_VM_PDB1: 55 + case AMDGPU_VM_PDB0: 56 + return 9 * (AMDGPU_VM_PDB0 - level) + 57 + adev->vm_manager.block_size; 58 + case AMDGPU_VM_PTB: 59 + return 0; 60 + default: 61 + return ~0; 62 + } 63 + } 64 + 65 + /** 66 + * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT 67 + * 68 + * @adev: amdgpu_device pointer 69 + * @level: VMPT level 70 + * 71 + * Returns: 72 + * The number of entries in a page directory or page table. 73 + */ 74 + static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev, 75 + unsigned int level) 76 + { 77 + unsigned int shift; 78 + 79 + shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); 80 + if (level == adev->vm_manager.root_level) 81 + /* For the root directory */ 82 + return round_up(adev->vm_manager.max_pfn, 1ULL << shift) 83 + >> shift; 84 + else if (level != AMDGPU_VM_PTB) 85 + /* Everything in between */ 86 + return 512; 87 + 88 + /* For the page tables on the leaves */ 89 + return AMDGPU_VM_PTE_COUNT(adev); 90 + } 91 + 92 + /** 93 + * amdgpu_vm_pt_num_ats_entries - return the number of ATS entries in the root PD 94 + * 95 + * @adev: amdgpu_device pointer 96 + * 97 + * Returns: 98 + * The number of entries in the root page directory which needs the ATS setting. 99 + */ 100 + static unsigned int amdgpu_vm_pt_num_ats_entries(struct amdgpu_device *adev) 101 + { 102 + unsigned int shift; 103 + 104 + shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); 105 + return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); 106 + } 107 + 108 + /** 109 + * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT 110 + * 111 + * @adev: amdgpu_device pointer 112 + * @level: VMPT level 113 + * 114 + * Returns: 115 + * The mask to extract the entry number of a PD/PT from an address. 116 + */ 117 + static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev, 118 + unsigned int level) 119 + { 120 + if (level <= adev->vm_manager.root_level) 121 + return 0xffffffff; 122 + else if (level != AMDGPU_VM_PTB) 123 + return 0x1ff; 124 + else 125 + return AMDGPU_VM_PTE_COUNT(adev) - 1; 126 + } 127 + 128 + /** 129 + * amdgpu_vm_pt_size - returns the size of the page table in bytes 130 + * 131 + * @adev: amdgpu_device pointer 132 + * @level: VMPT level 133 + * 134 + * Returns: 135 + * The size of the BO for a page directory or page table in bytes. 136 + */ 137 + static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev, 138 + unsigned int level) 139 + { 140 + return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8); 141 + } 142 + 143 + /** 144 + * amdgpu_vm_pt_parent - get the parent page directory 145 + * 146 + * @pt: child page table 147 + * 148 + * Helper to get the parent entry for the child page table. NULL if we are at 149 + * the root page directory. 150 + */ 151 + static struct amdgpu_vm_bo_base * 152 + amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt) 153 + { 154 + struct amdgpu_bo *parent = pt->bo->parent; 155 + 156 + if (!parent) 157 + return NULL; 158 + 159 + return parent->vm_bo; 160 + } 161 + 162 + /** 163 + * amdgpu_vm_pt_start - start PD/PT walk 164 + * 165 + * @adev: amdgpu_device pointer 166 + * @vm: amdgpu_vm structure 167 + * @start: start address of the walk 168 + * @cursor: state to initialize 169 + * 170 + * Initialize a amdgpu_vm_pt_cursor to start a walk. 171 + */ 172 + static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 173 + struct amdgpu_vm *vm, uint64_t start, 174 + struct amdgpu_vm_pt_cursor *cursor) 175 + { 176 + cursor->pfn = start; 177 + cursor->parent = NULL; 178 + cursor->entry = &vm->root; 179 + cursor->level = adev->vm_manager.root_level; 180 + } 181 + 182 + /** 183 + * amdgpu_vm_pt_descendant - go to child node 184 + * 185 + * @adev: amdgpu_device pointer 186 + * @cursor: current state 187 + * 188 + * Walk to the child node of the current node. 189 + * Returns: 190 + * True if the walk was possible, false otherwise. 191 + */ 192 + static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 193 + struct amdgpu_vm_pt_cursor *cursor) 194 + { 195 + unsigned int mask, shift, idx; 196 + 197 + if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || 198 + !cursor->entry->bo) 199 + return false; 200 + 201 + mask = amdgpu_vm_pt_entries_mask(adev, cursor->level); 202 + shift = amdgpu_vm_pt_level_shift(adev, cursor->level); 203 + 204 + ++cursor->level; 205 + idx = (cursor->pfn >> shift) & mask; 206 + cursor->parent = cursor->entry; 207 + cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; 208 + return true; 209 + } 210 + 211 + /** 212 + * amdgpu_vm_pt_sibling - go to sibling node 213 + * 214 + * @adev: amdgpu_device pointer 215 + * @cursor: current state 216 + * 217 + * Walk to the sibling node of the current node. 218 + * Returns: 219 + * True if the walk was possible, false otherwise. 220 + */ 221 + static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 222 + struct amdgpu_vm_pt_cursor *cursor) 223 + { 224 + 225 + unsigned int shift, num_entries; 226 + struct amdgpu_bo_vm *parent; 227 + 228 + /* Root doesn't have a sibling */ 229 + if (!cursor->parent) 230 + return false; 231 + 232 + /* Go to our parents and see if we got a sibling */ 233 + shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1); 234 + num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1); 235 + parent = to_amdgpu_bo_vm(cursor->parent->bo); 236 + 237 + if (cursor->entry == &parent->entries[num_entries - 1]) 238 + return false; 239 + 240 + cursor->pfn += 1ULL << shift; 241 + cursor->pfn &= ~((1ULL << shift) - 1); 242 + ++cursor->entry; 243 + return true; 244 + } 245 + 246 + /** 247 + * amdgpu_vm_pt_ancestor - go to parent node 248 + * 249 + * @cursor: current state 250 + * 251 + * Walk to the parent node of the current node. 252 + * Returns: 253 + * True if the walk was possible, false otherwise. 254 + */ 255 + static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 256 + { 257 + if (!cursor->parent) 258 + return false; 259 + 260 + --cursor->level; 261 + cursor->entry = cursor->parent; 262 + cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 263 + return true; 264 + } 265 + 266 + /** 267 + * amdgpu_vm_pt_next - get next PD/PT in hieratchy 268 + * 269 + * @adev: amdgpu_device pointer 270 + * @cursor: current state 271 + * 272 + * Walk the PD/PT tree to the next node. 273 + */ 274 + static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 275 + struct amdgpu_vm_pt_cursor *cursor) 276 + { 277 + /* First try a newborn child */ 278 + if (amdgpu_vm_pt_descendant(adev, cursor)) 279 + return; 280 + 281 + /* If that didn't worked try to find a sibling */ 282 + while (!amdgpu_vm_pt_sibling(adev, cursor)) { 283 + /* No sibling, go to our parents and grandparents */ 284 + if (!amdgpu_vm_pt_ancestor(cursor)) { 285 + cursor->pfn = ~0ll; 286 + return; 287 + } 288 + } 289 + } 290 + 291 + /** 292 + * amdgpu_vm_pt_first_dfs - start a deep first search 293 + * 294 + * @adev: amdgpu_device structure 295 + * @vm: amdgpu_vm structure 296 + * @start: optional cursor to start with 297 + * @cursor: state to initialize 298 + * 299 + * Starts a deep first traversal of the PD/PT tree. 300 + */ 301 + static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 302 + struct amdgpu_vm *vm, 303 + struct amdgpu_vm_pt_cursor *start, 304 + struct amdgpu_vm_pt_cursor *cursor) 305 + { 306 + if (start) 307 + *cursor = *start; 308 + else 309 + amdgpu_vm_pt_start(adev, vm, 0, cursor); 310 + 311 + while (amdgpu_vm_pt_descendant(adev, cursor)) 312 + ; 313 + } 314 + 315 + /** 316 + * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 317 + * 318 + * @start: starting point for the search 319 + * @entry: current entry 320 + * 321 + * Returns: 322 + * True when the search should continue, false otherwise. 323 + */ 324 + static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 325 + struct amdgpu_vm_bo_base *entry) 326 + { 327 + return entry && (!start || entry != start->entry); 328 + } 329 + 330 + /** 331 + * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 332 + * 333 + * @adev: amdgpu_device structure 334 + * @cursor: current state 335 + * 336 + * Move the cursor to the next node in a deep first search. 337 + */ 338 + static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 339 + struct amdgpu_vm_pt_cursor *cursor) 340 + { 341 + if (!cursor->entry) 342 + return; 343 + 344 + if (!cursor->parent) 345 + cursor->entry = NULL; 346 + else if (amdgpu_vm_pt_sibling(adev, cursor)) 347 + while (amdgpu_vm_pt_descendant(adev, cursor)) 348 + ; 349 + else 350 + amdgpu_vm_pt_ancestor(cursor); 351 + } 352 + 353 + /* 354 + * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 355 + */ 356 + #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 357 + for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 358 + (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 359 + amdgpu_vm_pt_continue_dfs((start), (entry)); \ 360 + (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 361 + 362 + /** 363 + * amdgpu_vm_pt_clear - initially clear the PDs/PTs 364 + * 365 + * @adev: amdgpu_device pointer 366 + * @vm: VM to clear BO from 367 + * @vmbo: BO to clear 368 + * @immediate: use an immediate update 369 + * 370 + * Root PD needs to be reserved when calling this. 371 + * 372 + * Returns: 373 + * 0 on success, errno otherwise. 374 + */ 375 + int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, 376 + struct amdgpu_bo_vm *vmbo, bool immediate) 377 + { 378 + unsigned int level = adev->vm_manager.root_level; 379 + struct ttm_operation_ctx ctx = { true, false }; 380 + struct amdgpu_vm_update_params params; 381 + struct amdgpu_bo *ancestor = &vmbo->bo; 382 + unsigned int entries, ats_entries; 383 + struct amdgpu_bo *bo = &vmbo->bo; 384 + uint64_t addr; 385 + int r, idx; 386 + 387 + /* Figure out our place in the hierarchy */ 388 + if (ancestor->parent) { 389 + ++level; 390 + while (ancestor->parent->parent) { 391 + ++level; 392 + ancestor = ancestor->parent; 393 + } 394 + } 395 + 396 + entries = amdgpu_bo_size(bo) / 8; 397 + if (!vm->pte_support_ats) { 398 + ats_entries = 0; 399 + 400 + } else if (!bo->parent) { 401 + ats_entries = amdgpu_vm_pt_num_ats_entries(adev); 402 + ats_entries = min(ats_entries, entries); 403 + entries -= ats_entries; 404 + 405 + } else { 406 + struct amdgpu_vm_bo_base *pt; 407 + 408 + pt = ancestor->vm_bo; 409 + ats_entries = amdgpu_vm_pt_num_ats_entries(adev); 410 + if ((pt - to_amdgpu_bo_vm(vm->root.bo)->entries) >= 411 + ats_entries) { 412 + ats_entries = 0; 413 + } else { 414 + ats_entries = entries; 415 + entries = 0; 416 + } 417 + } 418 + 419 + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 420 + if (r) 421 + return r; 422 + 423 + if (vmbo->shadow) { 424 + struct amdgpu_bo *shadow = vmbo->shadow; 425 + 426 + r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx); 427 + if (r) 428 + return r; 429 + } 430 + 431 + if (!drm_dev_enter(adev_to_drm(adev), &idx)) 432 + return -ENODEV; 433 + 434 + r = vm->update_funcs->map_table(vmbo); 435 + if (r) 436 + goto exit; 437 + 438 + memset(&params, 0, sizeof(params)); 439 + params.adev = adev; 440 + params.vm = vm; 441 + params.immediate = immediate; 442 + 443 + r = vm->update_funcs->prepare(&params, NULL, AMDGPU_SYNC_EXPLICIT); 444 + if (r) 445 + goto exit; 446 + 447 + addr = 0; 448 + if (ats_entries) { 449 + uint64_t value = 0, flags; 450 + 451 + flags = AMDGPU_PTE_DEFAULT_ATC; 452 + if (level != AMDGPU_VM_PTB) { 453 + /* Handle leaf PDEs as PTEs */ 454 + flags |= AMDGPU_PDE_PTE; 455 + amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); 456 + } 457 + 458 + r = vm->update_funcs->update(&params, vmbo, addr, 0, 459 + ats_entries, value, flags); 460 + if (r) 461 + goto exit; 462 + 463 + addr += ats_entries * 8; 464 + } 465 + 466 + if (entries) { 467 + uint64_t value = 0, flags = 0; 468 + 469 + if (adev->asic_type >= CHIP_VEGA10) { 470 + if (level != AMDGPU_VM_PTB) { 471 + /* Handle leaf PDEs as PTEs */ 472 + flags |= AMDGPU_PDE_PTE; 473 + amdgpu_gmc_get_vm_pde(adev, level, 474 + &value, &flags); 475 + } else { 476 + /* Workaround for fault priority problem on GMC9 */ 477 + flags = AMDGPU_PTE_EXECUTABLE; 478 + } 479 + } 480 + 481 + r = vm->update_funcs->update(&params, vmbo, addr, 0, entries, 482 + value, flags); 483 + if (r) 484 + goto exit; 485 + } 486 + 487 + r = vm->update_funcs->commit(&params, NULL); 488 + exit: 489 + drm_dev_exit(idx); 490 + return r; 491 + } 492 + 493 + /** 494 + * amdgpu_vm_pt_create - create bo for PD/PT 495 + * 496 + * @adev: amdgpu_device pointer 497 + * @vm: requesting vm 498 + * @level: the page table level 499 + * @immediate: use a immediate update 500 + * @vmbo: pointer to the buffer object pointer 501 + */ 502 + int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, 503 + int level, bool immediate, struct amdgpu_bo_vm **vmbo) 504 + { 505 + struct amdgpu_bo_param bp; 506 + struct amdgpu_bo *bo; 507 + struct dma_resv *resv; 508 + unsigned int num_entries; 509 + int r; 510 + 511 + memset(&bp, 0, sizeof(bp)); 512 + 513 + bp.size = amdgpu_vm_pt_size(adev, level); 514 + bp.byte_align = AMDGPU_GPU_PAGE_SIZE; 515 + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 516 + bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); 517 + bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 518 + AMDGPU_GEM_CREATE_CPU_GTT_USWC; 519 + 520 + if (level < AMDGPU_VM_PTB) 521 + num_entries = amdgpu_vm_pt_num_entries(adev, level); 522 + else 523 + num_entries = 0; 524 + 525 + bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries); 526 + 527 + if (vm->use_cpu_for_update) 528 + bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 529 + 530 + bp.type = ttm_bo_type_kernel; 531 + bp.no_wait_gpu = immediate; 532 + if (vm->root.bo) 533 + bp.resv = vm->root.bo->tbo.base.resv; 534 + 535 + r = amdgpu_bo_create_vm(adev, &bp, vmbo); 536 + if (r) 537 + return r; 538 + 539 + bo = &(*vmbo)->bo; 540 + if (vm->is_compute_context || (adev->flags & AMD_IS_APU)) { 541 + (*vmbo)->shadow = NULL; 542 + return 0; 543 + } 544 + 545 + if (!bp.resv) 546 + WARN_ON(dma_resv_lock(bo->tbo.base.resv, 547 + NULL)); 548 + resv = bp.resv; 549 + memset(&bp, 0, sizeof(bp)); 550 + bp.size = amdgpu_vm_pt_size(adev, level); 551 + bp.domain = AMDGPU_GEM_DOMAIN_GTT; 552 + bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; 553 + bp.type = ttm_bo_type_kernel; 554 + bp.resv = bo->tbo.base.resv; 555 + bp.bo_ptr_size = sizeof(struct amdgpu_bo); 556 + 557 + r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow); 558 + 559 + if (!resv) 560 + dma_resv_unlock(bo->tbo.base.resv); 561 + 562 + if (r) { 563 + amdgpu_bo_unref(&bo); 564 + return r; 565 + } 566 + 567 + (*vmbo)->shadow->parent = amdgpu_bo_ref(bo); 568 + amdgpu_bo_add_to_shadow_list(*vmbo); 569 + 570 + return 0; 571 + } 572 + 573 + /** 574 + * amdgpu_vm_pt_alloc - Allocate a specific page table 575 + * 576 + * @adev: amdgpu_device pointer 577 + * @vm: VM to allocate page tables for 578 + * @cursor: Which page table to allocate 579 + * @immediate: use an immediate update 580 + * 581 + * Make sure a specific page table or directory is allocated. 582 + * 583 + * Returns: 584 + * 1 if page table needed to be allocated, 0 if page table was already 585 + * allocated, negative errno if an error occurred. 586 + */ 587 + static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, 588 + struct amdgpu_vm *vm, 589 + struct amdgpu_vm_pt_cursor *cursor, 590 + bool immediate) 591 + { 592 + struct amdgpu_vm_bo_base *entry = cursor->entry; 593 + struct amdgpu_bo *pt_bo; 594 + struct amdgpu_bo_vm *pt; 595 + int r; 596 + 597 + if (entry->bo) 598 + return 0; 599 + 600 + r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt); 601 + if (r) 602 + return r; 603 + 604 + /* Keep a reference to the root directory to avoid 605 + * freeing them up in the wrong order. 606 + */ 607 + pt_bo = &pt->bo; 608 + pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); 609 + amdgpu_vm_bo_base_init(entry, vm, pt_bo); 610 + r = amdgpu_vm_pt_clear(adev, vm, pt, immediate); 611 + if (r) 612 + goto error_free_pt; 613 + 614 + return 0; 615 + 616 + error_free_pt: 617 + amdgpu_bo_unref(&pt->shadow); 618 + amdgpu_bo_unref(&pt_bo); 619 + return r; 620 + } 621 + 622 + /** 623 + * amdgpu_vm_free_table - fre one PD/PT 624 + * 625 + * @entry: PDE to free 626 + */ 627 + static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) 628 + { 629 + struct amdgpu_bo *shadow; 630 + 631 + if (!entry->bo) 632 + return; 633 + shadow = amdgpu_bo_shadowed(entry->bo); 634 + entry->bo->vm_bo = NULL; 635 + list_del(&entry->vm_status); 636 + amdgpu_bo_unref(&shadow); 637 + amdgpu_bo_unref(&entry->bo); 638 + } 639 + 640 + /** 641 + * amdgpu_vm_pt_free_dfs - free PD/PT levels 642 + * 643 + * @adev: amdgpu device structure 644 + * @vm: amdgpu vm structure 645 + * @start: optional cursor where to start freeing PDs/PTs 646 + * 647 + * Free the page directory or page table level and all sub levels. 648 + */ 649 + static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev, 650 + struct amdgpu_vm *vm, 651 + struct amdgpu_vm_pt_cursor *start) 652 + { 653 + struct amdgpu_vm_pt_cursor cursor; 654 + struct amdgpu_vm_bo_base *entry; 655 + 656 + vm->bulk_moveable = false; 657 + 658 + for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) 659 + amdgpu_vm_pt_free(entry); 660 + 661 + if (start) 662 + amdgpu_vm_pt_free(start->entry); 663 + } 664 + 665 + /** 666 + * amdgpu_vm_pt_free_root - free root PD 667 + * @adev: amdgpu device structure 668 + * @vm: amdgpu vm structure 669 + * 670 + * Free the root page directory and everything below it. 671 + */ 672 + void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm) 673 + { 674 + amdgpu_vm_pt_free_dfs(adev, vm, NULL); 675 + } 676 + 677 + /** 678 + * amdgpu_vm_pt_is_root_clean - check if a root PD is clean 679 + * 680 + * @adev: amdgpu_device pointer 681 + * @vm: the VM to check 682 + * 683 + * Check all entries of the root PD, if any subsequent PDs are allocated, 684 + * it means there are page table creating and filling, and is no a clean 685 + * VM 686 + * 687 + * Returns: 688 + * 0 if this VM is clean 689 + */ 690 + bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev, 691 + struct amdgpu_vm *vm) 692 + { 693 + enum amdgpu_vm_level root = adev->vm_manager.root_level; 694 + unsigned int entries = amdgpu_vm_pt_num_entries(adev, root); 695 + unsigned int i = 0; 696 + 697 + for (i = 0; i < entries; i++) { 698 + if (to_amdgpu_bo_vm(vm->root.bo)->entries[i].bo) 699 + return false; 700 + } 701 + return true; 702 + } 703 + 704 + /** 705 + * amdgpu_vm_pde_update - update a single level in the hierarchy 706 + * 707 + * @params: parameters for the update 708 + * @entry: entry to update 709 + * 710 + * Makes sure the requested entry in parent is up to date. 711 + */ 712 + int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 713 + struct amdgpu_vm_bo_base *entry) 714 + { 715 + struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry); 716 + struct amdgpu_bo *bo = parent->bo, *pbo; 717 + struct amdgpu_vm *vm = params->vm; 718 + uint64_t pde, pt, flags; 719 + unsigned int level; 720 + 721 + for (level = 0, pbo = bo->parent; pbo; ++level) 722 + pbo = pbo->parent; 723 + 724 + level += params->adev->vm_manager.root_level; 725 + amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags); 726 + pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8; 727 + return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt, 728 + 1, 0, flags); 729 + } 730 + 731 + /* 732 + * amdgpu_vm_pte_update_flags - figure out flags for PTE updates 733 + * 734 + * Make sure to set the right flags for the PTEs at the desired level. 735 + */ 736 + static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, 737 + struct amdgpu_bo_vm *pt, 738 + unsigned int level, 739 + uint64_t pe, uint64_t addr, 740 + unsigned int count, uint32_t incr, 741 + uint64_t flags) 742 + 743 + { 744 + if (level != AMDGPU_VM_PTB) { 745 + flags |= AMDGPU_PDE_PTE; 746 + amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); 747 + 748 + } else if (params->adev->asic_type >= CHIP_VEGA10 && 749 + !(flags & AMDGPU_PTE_VALID) && 750 + !(flags & AMDGPU_PTE_PRT)) { 751 + 752 + /* Workaround for fault priority problem on GMC9 */ 753 + flags |= AMDGPU_PTE_EXECUTABLE; 754 + } 755 + 756 + params->vm->update_funcs->update(params, pt, pe, addr, count, incr, 757 + flags); 758 + } 759 + 760 + /** 761 + * amdgpu_vm_pte_fragment - get fragment for PTEs 762 + * 763 + * @params: see amdgpu_vm_update_params definition 764 + * @start: first PTE to handle 765 + * @end: last PTE to handle 766 + * @flags: hw mapping flags 767 + * @frag: resulting fragment size 768 + * @frag_end: end of this fragment 769 + * 770 + * Returns the first possible fragment for the start and end address. 771 + */ 772 + static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params, 773 + uint64_t start, uint64_t end, uint64_t flags, 774 + unsigned int *frag, uint64_t *frag_end) 775 + { 776 + /** 777 + * The MC L1 TLB supports variable sized pages, based on a fragment 778 + * field in the PTE. When this field is set to a non-zero value, page 779 + * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 780 + * flags are considered valid for all PTEs within the fragment range 781 + * and corresponding mappings are assumed to be physically contiguous. 782 + * 783 + * The L1 TLB can store a single PTE for the whole fragment, 784 + * significantly increasing the space available for translation 785 + * caching. This leads to large improvements in throughput when the 786 + * TLB is under pressure. 787 + * 788 + * The L2 TLB distributes small and large fragments into two 789 + * asymmetric partitions. The large fragment cache is significantly 790 + * larger. Thus, we try to use large fragments wherever possible. 791 + * Userspace can support this by aligning virtual base address and 792 + * allocation size to the fragment size. 793 + * 794 + * Starting with Vega10 the fragment size only controls the L1. The L2 795 + * is now directly feed with small/huge/giant pages from the walker. 796 + */ 797 + unsigned int max_frag; 798 + 799 + if (params->adev->asic_type < CHIP_VEGA10) 800 + max_frag = params->adev->vm_manager.fragment_size; 801 + else 802 + max_frag = 31; 803 + 804 + /* system pages are non continuously */ 805 + if (params->pages_addr) { 806 + *frag = 0; 807 + *frag_end = end; 808 + return; 809 + } 810 + 811 + /* This intentionally wraps around if no bit is set */ 812 + *frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1); 813 + if (*frag >= max_frag) { 814 + *frag = max_frag; 815 + *frag_end = end & ~((1ULL << max_frag) - 1); 816 + } else { 817 + *frag_end = start + (1 << *frag); 818 + } 819 + } 820 + 821 + /** 822 + * amdgpu_vm_ptes_update - make sure that page tables are valid 823 + * 824 + * @params: see amdgpu_vm_update_params definition 825 + * @start: start of GPU address range 826 + * @end: end of GPU address range 827 + * @dst: destination address to map to, the next dst inside the function 828 + * @flags: mapping flags 829 + * 830 + * Update the page tables in the range @start - @end. 831 + * 832 + * Returns: 833 + * 0 for success, -EINVAL for failure. 834 + */ 835 + int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, 836 + uint64_t start, uint64_t end, 837 + uint64_t dst, uint64_t flags) 838 + { 839 + struct amdgpu_device *adev = params->adev; 840 + struct amdgpu_vm_pt_cursor cursor; 841 + uint64_t frag_start = start, frag_end; 842 + unsigned int frag; 843 + int r; 844 + 845 + /* figure out the initial fragment */ 846 + amdgpu_vm_pte_fragment(params, frag_start, end, flags, &frag, 847 + &frag_end); 848 + 849 + /* walk over the address space and update the PTs */ 850 + amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 851 + while (cursor.pfn < end) { 852 + unsigned int shift, parent_shift, mask; 853 + uint64_t incr, entry_end, pe_start; 854 + struct amdgpu_bo *pt; 855 + 856 + if (!params->unlocked) { 857 + /* make sure that the page tables covering the 858 + * address range are actually allocated 859 + */ 860 + r = amdgpu_vm_pt_alloc(params->adev, params->vm, 861 + &cursor, params->immediate); 862 + if (r) 863 + return r; 864 + } 865 + 866 + shift = amdgpu_vm_pt_level_shift(adev, cursor.level); 867 + parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1); 868 + if (params->unlocked) { 869 + /* Unlocked updates are only allowed on the leaves */ 870 + if (amdgpu_vm_pt_descendant(adev, &cursor)) 871 + continue; 872 + } else if (adev->asic_type < CHIP_VEGA10 && 873 + (flags & AMDGPU_PTE_VALID)) { 874 + /* No huge page support before GMC v9 */ 875 + if (cursor.level != AMDGPU_VM_PTB) { 876 + if (!amdgpu_vm_pt_descendant(adev, &cursor)) 877 + return -ENOENT; 878 + continue; 879 + } 880 + } else if (frag < shift) { 881 + /* We can't use this level when the fragment size is 882 + * smaller than the address shift. Go to the next 883 + * child entry and try again. 884 + */ 885 + if (amdgpu_vm_pt_descendant(adev, &cursor)) 886 + continue; 887 + } else if (frag >= parent_shift) { 888 + /* If the fragment size is even larger than the parent 889 + * shift we should go up one level and check it again. 890 + */ 891 + if (!amdgpu_vm_pt_ancestor(&cursor)) 892 + return -EINVAL; 893 + continue; 894 + } 895 + 896 + pt = cursor.entry->bo; 897 + if (!pt) { 898 + /* We need all PDs and PTs for mapping something, */ 899 + if (flags & AMDGPU_PTE_VALID) 900 + return -ENOENT; 901 + 902 + /* but unmapping something can happen at a higher 903 + * level. 904 + */ 905 + if (!amdgpu_vm_pt_ancestor(&cursor)) 906 + return -EINVAL; 907 + 908 + pt = cursor.entry->bo; 909 + shift = parent_shift; 910 + frag_end = max(frag_end, ALIGN(frag_start + 1, 911 + 1ULL << shift)); 912 + } 913 + 914 + /* Looks good so far, calculate parameters for the update */ 915 + incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 916 + mask = amdgpu_vm_pt_entries_mask(adev, cursor.level); 917 + pe_start = ((cursor.pfn >> shift) & mask) * 8; 918 + entry_end = ((uint64_t)mask + 1) << shift; 919 + entry_end += cursor.pfn & ~(entry_end - 1); 920 + entry_end = min(entry_end, end); 921 + 922 + do { 923 + struct amdgpu_vm *vm = params->vm; 924 + uint64_t upd_end = min(entry_end, frag_end); 925 + unsigned int nptes = (upd_end - frag_start) >> shift; 926 + uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag); 927 + 928 + /* This can happen when we set higher level PDs to 929 + * silent to stop fault floods. 930 + */ 931 + nptes = max(nptes, 1u); 932 + 933 + trace_amdgpu_vm_update_ptes(params, frag_start, upd_end, 934 + min(nptes, 32u), dst, incr, 935 + upd_flags, 936 + vm->task_info.pid, 937 + vm->immediate.fence_context); 938 + amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt), 939 + cursor.level, pe_start, dst, 940 + nptes, incr, upd_flags); 941 + 942 + pe_start += nptes * 8; 943 + dst += nptes * incr; 944 + 945 + frag_start = upd_end; 946 + if (frag_start >= frag_end) { 947 + /* figure out the next fragment */ 948 + amdgpu_vm_pte_fragment(params, frag_start, end, 949 + flags, &frag, &frag_end); 950 + if (frag < shift) 951 + break; 952 + } 953 + } while (frag_start < entry_end); 954 + 955 + if (amdgpu_vm_pt_descendant(adev, &cursor)) { 956 + /* Free all child entries. 957 + * Update the tables with the flags and addresses and free up subsequent 958 + * tables in the case of huge pages or freed up areas. 959 + * This is the maximum you can free, because all other page tables are not 960 + * completely covered by the range and so potentially still in use. 961 + */ 962 + while (cursor.pfn < frag_start) { 963 + /* Make sure previous mapping is freed */ 964 + if (cursor.entry->bo) { 965 + params->table_freed = true; 966 + amdgpu_vm_pt_free_dfs(adev, params->vm, 967 + &cursor); 968 + } 969 + amdgpu_vm_pt_next(adev, &cursor); 970 + } 971 + 972 + } else if (frag >= shift) { 973 + /* or just move on to the next on the same level. */ 974 + amdgpu_vm_pt_next(adev, &cursor); 975 + } 976 + } 977 + 978 + return 0; 979 + }

Configure Feed

Configure Feed