drm/msm/a6xx: Implement preemption for a7xx targets

+1

drivers/gpu/drm/msm/Makefile

··· 23 23 adreno/a6xx_gpu.o \ 24 24 adreno/a6xx_gmu.o \ 25 25 adreno/a6xx_hfi.o \ 26 + adreno/a6xx_preempt.o \ 26 27 27 28 adreno-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o \ 28 29

+182 -11

drivers/gpu/drm/msm/adreno/a6xx_gpu.c

··· 68 68 69 69 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 70 70 { 71 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 72 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 71 73 uint32_t wptr; 72 74 unsigned long flags; 73 75 ··· 83 81 /* Make sure to wrap wptr if we need to */ 84 82 wptr = get_wptr(ring); 85 83 84 + /* Update HW if this is the current ring and we are not in preempt*/ 85 + if (!a6xx_in_preempt(a6xx_gpu)) { 86 + if (a6xx_gpu->cur_ring == ring) 87 + gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); 88 + else 89 + ring->restore_wptr = true; 90 + } else { 91 + ring->restore_wptr = true; 92 + } 93 + 86 94 spin_unlock_irqrestore(&ring->preempt_lock, flags); 87 - 88 - /* Make sure everything is posted before making a decision */ 89 - mb(); 90 - 91 - gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); 92 95 } 93 96 94 97 static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter, ··· 145 138 146 139 /* 147 140 * Write the new TTBR0 to the memstore. This is good for debugging. 141 + * Needed for preemption 148 142 */ 149 - OUT_PKT7(ring, CP_MEM_WRITE, 4); 143 + OUT_PKT7(ring, CP_MEM_WRITE, 5); 150 144 OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr))); 151 145 OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr))); 152 146 OUT_RING(ring, lower_32_bits(ttbr)); 153 - OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr)); 147 + OUT_RING(ring, upper_32_bits(ttbr)); 148 + OUT_RING(ring, ctx->seqno); 154 149 155 150 /* 156 151 * Sync both threads after switching pagetables and enable BR only ··· 277 268 a6xx_flush(gpu, ring); 278 269 } 279 270 271 + static void a6xx_emit_set_pseudo_reg(struct msm_ringbuffer *ring, 272 + struct a6xx_gpu *a6xx_gpu, struct msm_gpu_submitqueue *queue) 273 + { 274 + OUT_PKT7(ring, CP_SET_PSEUDO_REG, 12); 275 + 276 + OUT_RING(ring, SMMU_INFO); 277 + /* don't save SMMU, we write the record from the kernel instead */ 278 + OUT_RING(ring, 0); 279 + OUT_RING(ring, 0); 280 + 281 + /* privileged and non secure buffer save */ 282 + OUT_RING(ring, NON_SECURE_SAVE_ADDR); 283 + OUT_RING(ring, lower_32_bits( 284 + a6xx_gpu->preempt_iova[ring->id])); 285 + OUT_RING(ring, upper_32_bits( 286 + a6xx_gpu->preempt_iova[ring->id])); 287 + 288 + /* user context buffer save, seems to be unnused by fw */ 289 + OUT_RING(ring, NON_PRIV_SAVE_ADDR); 290 + OUT_RING(ring, 0); 291 + OUT_RING(ring, 0); 292 + 293 + OUT_RING(ring, COUNTER); 294 + /* seems OK to set to 0 to disable it */ 295 + OUT_RING(ring, 0); 296 + OUT_RING(ring, 0); 297 + } 298 + 280 299 static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) 281 300 { 282 301 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; ··· 321 284 OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR); 322 285 323 286 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx); 287 + 288 + /* 289 + * If preemption is enabled, then set the pseudo register for the save 290 + * sequence 291 + */ 292 + if (gpu->nr_rings > 1) 293 + a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, submit->queue); 324 294 325 295 get_stats_counter(ring, REG_A7XX_RBBM_PERFCTR_CP(0), 326 296 rbmemptr_stats(ring, index, cpcycles_start)); ··· 420 376 OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence))); 421 377 OUT_RING(ring, submit->seqno); 422 378 379 + a6xx_gpu->last_seqno[ring->id] = submit->seqno; 380 + 423 381 /* write the ringbuffer timestamp */ 424 382 OUT_PKT7(ring, CP_EVENT_WRITE, 4); 425 383 OUT_RING(ring, CACHE_CLEAN | CP_EVENT_WRITE_0_IRQ | BIT(27)); ··· 435 389 OUT_PKT7(ring, CP_SET_MARKER, 1); 436 390 OUT_RING(ring, 0x100); /* IFPC enable */ 437 391 392 + /* If preemption is enabled */ 393 + if (gpu->nr_rings > 1) { 394 + /* Yield the floor on command completion */ 395 + OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 396 + 397 + /* 398 + * If dword[2:1] are non zero, they specify an address for 399 + * the CP to write the value of dword[3] to on preemption 400 + * complete. Write 0 to skip the write 401 + */ 402 + OUT_RING(ring, 0x00); 403 + OUT_RING(ring, 0x00); 404 + /* Data value - not used if the address above is 0 */ 405 + OUT_RING(ring, 0x01); 406 + /* generate interrupt on preemption completion */ 407 + OUT_RING(ring, 0x00); 408 + } 409 + 410 + 438 411 trace_msm_gpu_submit_flush(submit, 439 412 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER)); 440 413 441 414 a6xx_flush(gpu, ring); 415 + 416 + /* Check to see if we need to start preemption */ 417 + a6xx_preempt_trigger(gpu); 442 418 } 443 419 444 420 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state) ··· 667 599 adreno_gpu->ubwc_config.macrotile_mode); 668 600 } 669 601 602 + static void a7xx_patch_pwrup_reglist(struct msm_gpu *gpu) 603 + { 604 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 605 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 606 + const struct adreno_reglist_list *reglist; 607 + void *ptr = a6xx_gpu->pwrup_reglist_ptr; 608 + struct cpu_gpu_lock *lock = ptr; 609 + u32 *dest = (u32 *)&lock->regs[0]; 610 + int i; 611 + 612 + reglist = adreno_gpu->info->a6xx->pwrup_reglist; 613 + 614 + lock->gpu_req = lock->cpu_req = lock->turn = 0; 615 + lock->ifpc_list_len = 0; 616 + lock->preemption_list_len = reglist->count; 617 + 618 + /* 619 + * For each entry in each of the lists, write the offset and the current 620 + * register value into the GPU buffer 621 + */ 622 + for (i = 0; i < reglist->count; i++) { 623 + *dest++ = reglist->regs[i]; 624 + *dest++ = gpu_read(gpu, reglist->regs[i]); 625 + } 626 + 627 + /* 628 + * The overall register list is composed of 629 + * 1. Static IFPC-only registers 630 + * 2. Static IFPC + preemption registers 631 + * 3. Dynamic IFPC + preemption registers (ex: perfcounter selects) 632 + * 633 + * The first two lists are static. Size of these lists are stored as 634 + * number of pairs in ifpc_list_len and preemption_list_len 635 + * respectively. With concurrent binning, Some of the perfcounter 636 + * registers being virtualized, CP needs to know the pipe id to program 637 + * the aperture inorder to restore the same. Thus, third list is a 638 + * dynamic list with triplets as 639 + * (<aperture, shifted 12 bits> <address> <data>), and the length is 640 + * stored as number for triplets in dynamic_list_len. 641 + */ 642 + lock->dynamic_list_len = 0; 643 + } 644 + 645 + static int a7xx_preempt_start(struct msm_gpu *gpu) 646 + { 647 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 648 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 649 + struct msm_ringbuffer *ring = gpu->rb[0]; 650 + 651 + if (gpu->nr_rings <= 1) 652 + return 0; 653 + 654 + /* Turn CP protection off */ 655 + OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1); 656 + OUT_RING(ring, 0); 657 + 658 + a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, NULL); 659 + 660 + /* Yield the floor on command completion */ 661 + OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4); 662 + OUT_RING(ring, 0x00); 663 + OUT_RING(ring, 0x00); 664 + OUT_RING(ring, 0x00); 665 + /* Generate interrupt on preemption completion */ 666 + OUT_RING(ring, 0x00); 667 + 668 + a6xx_flush(gpu, ring); 669 + 670 + return a6xx_idle(gpu, ring) ? 0 : -EINVAL; 671 + } 672 + 670 673 static int a6xx_cp_init(struct msm_gpu *gpu) 671 674 { 672 675 struct msm_ringbuffer *ring = gpu->rb[0]; ··· 769 630 770 631 static int a7xx_cp_init(struct msm_gpu *gpu) 771 632 { 633 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 634 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 772 635 struct msm_ringbuffer *ring = gpu->rb[0]; 773 636 u32 mask; 774 637 ··· 808 667 809 668 /* *Don't* send a power up reg list for concurrent binning (TODO) */ 810 669 /* Lo address */ 811 - OUT_RING(ring, 0x00000000); 670 + OUT_RING(ring, lower_32_bits(a6xx_gpu->pwrup_reglist_iova)); 812 671 /* Hi address */ 813 - OUT_RING(ring, 0x00000000); 672 + OUT_RING(ring, upper_32_bits(a6xx_gpu->pwrup_reglist_iova)); 814 673 /* BIT(31) set => read the regs from the list */ 815 - OUT_RING(ring, 0x00000000); 674 + OUT_RING(ring, BIT(31)); 816 675 817 676 a6xx_flush(gpu, ring); 818 677 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; ··· 935 794 936 795 msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow"); 937 796 } 797 + 798 + a6xx_gpu->pwrup_reglist_ptr = msm_gem_kernel_new(gpu->dev, PAGE_SIZE, 799 + MSM_BO_WC | MSM_BO_MAP_PRIV, 800 + gpu->aspace, &a6xx_gpu->pwrup_reglist_bo, 801 + &a6xx_gpu->pwrup_reglist_iova); 802 + 803 + if (IS_ERR(a6xx_gpu->pwrup_reglist_ptr)) 804 + return PTR_ERR(a6xx_gpu->pwrup_reglist_ptr); 805 + 806 + msm_gem_object_set_name(a6xx_gpu->pwrup_reglist_bo, "pwrup_reglist"); 938 807 939 808 return 0; 940 809 } ··· 1276 1125 if (a6xx_gpu->shadow_bo) { 1277 1126 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR, 1278 1127 shadowptr(a6xx_gpu, gpu->rb[0])); 1128 + for (unsigned int i = 0; i < gpu->nr_rings; i++) 1129 + a6xx_gpu->shadow[i] = 0; 1279 1130 } 1280 1131 1281 1132 /* ..which means "always" on A7xx, also for BV shadow */ ··· 1285 1132 gpu_write64(gpu, REG_A7XX_CP_BV_RB_RPTR_ADDR, 1286 1133 rbmemptr(gpu->rb[0], bv_rptr)); 1287 1134 } 1135 + 1136 + a6xx_preempt_hw_init(gpu); 1288 1137 1289 1138 /* Always come up on rb 0 */ 1290 1139 a6xx_gpu->cur_ring = gpu->rb[0]; ··· 1296 1141 1297 1142 /* Enable the SQE_to start the CP engine */ 1298 1143 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1); 1144 + 1145 + if (adreno_is_a7xx(adreno_gpu) && !a6xx_gpu->pwrup_reglist_emitted) { 1146 + a7xx_patch_pwrup_reglist(gpu); 1147 + a6xx_gpu->pwrup_reglist_emitted = true; 1148 + } 1299 1149 1300 1150 ret = adreno_is_a7xx(adreno_gpu) ? a7xx_cp_init(gpu) : a6xx_cp_init(gpu); 1301 1151 if (ret) ··· 1339 1179 out: 1340 1180 if (adreno_has_gmu_wrapper(adreno_gpu)) 1341 1181 return ret; 1182 + 1183 + /* Last step - yield the ringbuffer */ 1184 + a7xx_preempt_start(gpu); 1185 + 1342 1186 /* 1343 1187 * Tell the GMU that we are done touching the GPU and it can start power 1344 1188 * management ··· 1720 1556 if (status & A6XX_RBBM_INT_0_MASK_SWFUSEVIOLATION) 1721 1557 a7xx_sw_fuse_violation_irq(gpu); 1722 1558 1723 - if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) 1559 + if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) { 1724 1560 msm_gpu_retire(gpu); 1561 + a6xx_preempt_trigger(gpu); 1562 + } 1563 + 1564 + if (status & A6XX_RBBM_INT_0_MASK_CP_SW) 1565 + a6xx_preempt_irq(gpu); 1725 1566 1726 1567 return IRQ_HANDLED; 1727 1568 } ··· 2499 2330 a6xx_fault_handler); 2500 2331 2501 2332 a6xx_calc_ubwc_config(adreno_gpu); 2333 + /* Set up the preemption specific bits and pieces for each ringbuffer */ 2334 + a6xx_preempt_init(gpu); 2502 2335 2503 2336 return gpu; 2504 2337 }

+162

drivers/gpu/drm/msm/adreno/a6xx_gpu.h

··· 12 12 13 13 extern bool hang_debug; 14 14 15 + struct cpu_gpu_lock { 16 + uint32_t gpu_req; 17 + uint32_t cpu_req; 18 + uint32_t turn; 19 + union { 20 + struct { 21 + uint16_t list_length; 22 + uint16_t list_offset; 23 + }; 24 + struct { 25 + uint8_t ifpc_list_len; 26 + uint8_t preemption_list_len; 27 + uint16_t dynamic_list_len; 28 + }; 29 + }; 30 + uint64_t regs[62]; 31 + }; 32 + 15 33 /** 16 34 * struct a6xx_info - a6xx specific information from device table 17 35 * ··· 53 35 uint64_t sqe_iova; 54 36 55 37 struct msm_ringbuffer *cur_ring; 38 + struct msm_ringbuffer *next_ring; 39 + 40 + struct drm_gem_object *preempt_bo[MSM_GPU_MAX_RINGS]; 41 + void *preempt[MSM_GPU_MAX_RINGS]; 42 + uint64_t preempt_iova[MSM_GPU_MAX_RINGS]; 43 + struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS]; 44 + void *preempt_smmu[MSM_GPU_MAX_RINGS]; 45 + uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS]; 46 + uint32_t last_seqno[MSM_GPU_MAX_RINGS]; 47 + 48 + atomic_t preempt_state; 49 + spinlock_t eval_lock; 50 + struct timer_list preempt_timer; 51 + 52 + unsigned int preempt_level; 53 + bool uses_gmem; 54 + bool skip_save_restore; 56 55 57 56 struct a6xx_gmu gmu; 58 57 59 58 struct drm_gem_object *shadow_bo; 60 59 uint64_t shadow_iova; 61 60 uint32_t *shadow; 61 + 62 + struct drm_gem_object *pwrup_reglist_bo; 63 + void *pwrup_reglist_ptr; 64 + uint64_t pwrup_reglist_iova; 65 + bool pwrup_reglist_emitted; 62 66 63 67 bool has_whereami; 64 68 ··· 92 52 }; 93 53 94 54 #define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) 55 + 56 + /* 57 + * In order to do lockless preemption we use a simple state machine to progress 58 + * through the process. 59 + * 60 + * PREEMPT_NONE - no preemption in progress. Next state START. 61 + * PREEMPT_START - The trigger is evaluating if preemption is possible. Next 62 + * states: TRIGGERED, NONE 63 + * PREEMPT_FINISH - An intermediate state before moving back to NONE. Next 64 + * state: NONE. 65 + * PREEMPT_TRIGGERED: A preemption has been executed on the hardware. Next 66 + * states: FAULTED, PENDING 67 + * PREEMPT_FAULTED: A preemption timed out (never completed). This will trigger 68 + * recovery. Next state: N/A 69 + * PREEMPT_PENDING: Preemption complete interrupt fired - the callback is 70 + * checking the success of the operation. Next state: FAULTED, NONE. 71 + */ 72 + 73 + enum a6xx_preempt_state { 74 + PREEMPT_NONE = 0, 75 + PREEMPT_START, 76 + PREEMPT_FINISH, 77 + PREEMPT_TRIGGERED, 78 + PREEMPT_FAULTED, 79 + PREEMPT_PENDING, 80 + }; 81 + 82 + /* 83 + * struct a6xx_preempt_record is a shared buffer between the microcode and the 84 + * CPU to store the state for preemption. The record itself is much larger 85 + * (2112k) but most of that is used by the CP for storage. 86 + * 87 + * There is a preemption record assigned per ringbuffer. When the CPU triggers a 88 + * preemption, it fills out the record with the useful information (wptr, ring 89 + * base, etc) and the microcode uses that information to set up the CP following 90 + * the preemption. When a ring is switched out, the CP will save the ringbuffer 91 + * state back to the record. In this way, once the records are properly set up 92 + * the CPU can quickly switch back and forth between ringbuffers by only 93 + * updating a few registers (often only the wptr). 94 + * 95 + * These are the CPU aware registers in the record: 96 + * @magic: Must always be 0xAE399D6EUL 97 + * @info: Type of the record - written 0 by the CPU, updated by the CP 98 + * @errno: preemption error record 99 + * @data: Data field in YIELD and SET_MARKER packets, Written and used by CP 100 + * @cntl: Value of RB_CNTL written by CPU, save/restored by CP 101 + * @rptr: Value of RB_RPTR written by CPU, save/restored by CP 102 + * @wptr: Value of RB_WPTR written by CPU, save/restored by CP 103 + * @_pad: Reserved/padding 104 + * @rptr_addr: Value of RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 105 + * @rbase: Value of RB_BASE written by CPU, save/restored by CP 106 + * @counter: GPU address of the storage area for the preemption counters 107 + * @bv_rptr_addr: Value of BV_RB_RPTR_ADDR_LO|HI written by CPU, save/restored by CP 108 + */ 109 + struct a6xx_preempt_record { 110 + u32 magic; 111 + u32 info; 112 + u32 errno; 113 + u32 data; 114 + u32 cntl; 115 + u32 rptr; 116 + u32 wptr; 117 + u32 _pad; 118 + u64 rptr_addr; 119 + u64 rbase; 120 + u64 counter; 121 + u64 bv_rptr_addr; 122 + }; 123 + 124 + #define A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6EUL 125 + 126 + #define PREEMPT_SMMU_INFO_SIZE 4096 127 + 128 + #define PREEMPT_RECORD_SIZE(adreno_gpu) \ 129 + ((adreno_gpu->info->preempt_record_size) == 0 ? \ 130 + 4192 * SZ_1K : (adreno_gpu->info->preempt_record_size)) 131 + 132 + /* 133 + * The preemption counter block is a storage area for the value of the 134 + * preemption counters that are saved immediately before context switch. We 135 + * append it on to the end of the allocation for the preemption record. 136 + */ 137 + #define A6XX_PREEMPT_COUNTER_SIZE (16 * 4) 138 + 139 + struct a7xx_cp_smmu_info { 140 + u32 magic; 141 + u32 _pad4; 142 + u64 ttbr0; 143 + u32 asid; 144 + u32 context_idr; 145 + u32 context_bank; 146 + }; 147 + 148 + #define GEN7_CP_SMMU_INFO_MAGIC 0x241350d5UL 95 149 96 150 /* 97 151 * Given a register and a count, return a value to program into ··· 243 109 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 244 110 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node); 245 111 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu); 112 + 113 + void a6xx_preempt_init(struct msm_gpu *gpu); 114 + void a6xx_preempt_hw_init(struct msm_gpu *gpu); 115 + void a6xx_preempt_trigger(struct msm_gpu *gpu); 116 + void a6xx_preempt_irq(struct msm_gpu *gpu); 117 + void a6xx_preempt_fini(struct msm_gpu *gpu); 118 + int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu, 119 + struct msm_gpu_submitqueue *queue); 120 + void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu, 121 + struct msm_gpu_submitqueue *queue); 122 + 123 + /* Return true if we are in a preempt state */ 124 + static inline bool a6xx_in_preempt(struct a6xx_gpu *a6xx_gpu) 125 + { 126 + /* 127 + * Make sure the read to preempt_state is ordered with respect to reads 128 + * of other variables before ... 129 + */ 130 + smp_rmb(); 131 + 132 + int preempt_state = atomic_read(&a6xx_gpu->preempt_state); 133 + 134 + /* ... and after. */ 135 + smp_rmb(); 136 + 137 + return !(preempt_state == PREEMPT_NONE || 138 + preempt_state == PREEMPT_FINISH); 139 + } 246 140 247 141 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, 248 142 bool suspended);

+393

drivers/gpu/drm/msm/adreno/a6xx_preempt.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* Copyright (c) 2018, The Linux Foundation. All rights reserved. */ 3 + /* Copyright (c) 2023 Collabora, Ltd. */ 4 + /* Copyright (c) 2024 Valve Corporation */ 5 + 6 + #include "msm_gem.h" 7 + #include "a6xx_gpu.h" 8 + #include "a6xx_gmu.xml.h" 9 + #include "msm_mmu.h" 10 + 11 + /* 12 + * Try to transition the preemption state from old to new. Return 13 + * true on success or false if the original state wasn't 'old' 14 + */ 15 + static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu, 16 + enum a6xx_preempt_state old, enum a6xx_preempt_state new) 17 + { 18 + enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state, 19 + old, new); 20 + 21 + return (cur == old); 22 + } 23 + 24 + /* 25 + * Force the preemption state to the specified state. This is used in cases 26 + * where the current state is known and won't change 27 + */ 28 + static inline void set_preempt_state(struct a6xx_gpu *gpu, 29 + enum a6xx_preempt_state new) 30 + { 31 + /* 32 + * preempt_state may be read by other cores trying to trigger a 33 + * preemption or in the interrupt handler so barriers are needed 34 + * before... 35 + */ 36 + smp_mb__before_atomic(); 37 + atomic_set(&gpu->preempt_state, new); 38 + /* ... and after*/ 39 + smp_mb__after_atomic(); 40 + } 41 + 42 + /* Write the most recent wptr for the given ring into the hardware */ 43 + static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 44 + { 45 + unsigned long flags; 46 + uint32_t wptr; 47 + 48 + spin_lock_irqsave(&ring->preempt_lock, flags); 49 + 50 + if (ring->restore_wptr) { 51 + wptr = get_wptr(ring); 52 + 53 + gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); 54 + 55 + ring->restore_wptr = false; 56 + } 57 + 58 + spin_unlock_irqrestore(&ring->preempt_lock, flags); 59 + } 60 + 61 + /* Return the highest priority ringbuffer with something in it */ 62 + static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) 63 + { 64 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 65 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 66 + 67 + unsigned long flags; 68 + int i; 69 + 70 + for (i = 0; i < gpu->nr_rings; i++) { 71 + bool empty; 72 + struct msm_ringbuffer *ring = gpu->rb[i]; 73 + 74 + spin_lock_irqsave(&ring->preempt_lock, flags); 75 + empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring)); 76 + if (!empty && ring == a6xx_gpu->cur_ring) 77 + empty = ring->memptrs->fence == a6xx_gpu->last_seqno[i]; 78 + spin_unlock_irqrestore(&ring->preempt_lock, flags); 79 + 80 + if (!empty) 81 + return ring; 82 + } 83 + 84 + return NULL; 85 + } 86 + 87 + static void a6xx_preempt_timer(struct timer_list *t) 88 + { 89 + struct a6xx_gpu *a6xx_gpu = from_timer(a6xx_gpu, t, preempt_timer); 90 + struct msm_gpu *gpu = &a6xx_gpu->base.base; 91 + struct drm_device *dev = gpu->dev; 92 + 93 + if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 94 + return; 95 + 96 + dev_err(dev->dev, "%s: preemption timed out\n", gpu->name); 97 + kthread_queue_work(gpu->worker, &gpu->recover_work); 98 + } 99 + 100 + void a6xx_preempt_irq(struct msm_gpu *gpu) 101 + { 102 + uint32_t status; 103 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 104 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 105 + struct drm_device *dev = gpu->dev; 106 + 107 + if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 108 + return; 109 + 110 + /* Delete the preemption watchdog timer */ 111 + del_timer(&a6xx_gpu->preempt_timer); 112 + 113 + /* 114 + * The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL 115 + * to zero before firing the interrupt, but there is a non zero chance 116 + * of a hardware condition or a software race that could set it again 117 + * before we have a chance to finish. If that happens, log and go for 118 + * recovery 119 + */ 120 + status = gpu_read(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL); 121 + if (unlikely(status & A6XX_CP_CONTEXT_SWITCH_CNTL_STOP)) { 122 + DRM_DEV_ERROR(&gpu->pdev->dev, 123 + "!!!!!!!!!!!!!!!! preemption faulted !!!!!!!!!!!!!! irq\n"); 124 + set_preempt_state(a6xx_gpu, PREEMPT_FAULTED); 125 + dev_err(dev->dev, "%s: Preemption failed to complete\n", 126 + gpu->name); 127 + kthread_queue_work(gpu->worker, &gpu->recover_work); 128 + return; 129 + } 130 + 131 + a6xx_gpu->cur_ring = a6xx_gpu->next_ring; 132 + a6xx_gpu->next_ring = NULL; 133 + 134 + set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 135 + 136 + update_wptr(gpu, a6xx_gpu->cur_ring); 137 + 138 + set_preempt_state(a6xx_gpu, PREEMPT_NONE); 139 + 140 + /* 141 + * Retrigger preemption to avoid a deadlock that might occur when preemption 142 + * is skipped due to it being already in flight when requested. 143 + */ 144 + a6xx_preempt_trigger(gpu); 145 + } 146 + 147 + void a6xx_preempt_hw_init(struct msm_gpu *gpu) 148 + { 149 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 150 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 151 + int i; 152 + 153 + /* No preemption if we only have one ring */ 154 + if (gpu->nr_rings == 1) 155 + return; 156 + 157 + for (i = 0; i < gpu->nr_rings; i++) { 158 + struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[i]; 159 + 160 + record_ptr->wptr = 0; 161 + record_ptr->rptr = 0; 162 + record_ptr->rptr_addr = shadowptr(a6xx_gpu, gpu->rb[i]); 163 + record_ptr->info = 0; 164 + record_ptr->data = 0; 165 + record_ptr->rbase = gpu->rb[i]->iova; 166 + } 167 + 168 + /* Write a 0 to signal that we aren't switching pagetables */ 169 + gpu_write64(gpu, REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, 0); 170 + 171 + /* Enable the GMEM save/restore feature for preemption */ 172 + gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, 0x1); 173 + 174 + /* Reset the preemption state */ 175 + set_preempt_state(a6xx_gpu, PREEMPT_NONE); 176 + 177 + spin_lock_init(&a6xx_gpu->eval_lock); 178 + 179 + /* Always come up on rb 0 */ 180 + a6xx_gpu->cur_ring = gpu->rb[0]; 181 + } 182 + 183 + void a6xx_preempt_trigger(struct msm_gpu *gpu) 184 + { 185 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 186 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 187 + unsigned long flags; 188 + struct msm_ringbuffer *ring; 189 + unsigned int cntl; 190 + 191 + if (gpu->nr_rings == 1) 192 + return; 193 + 194 + /* 195 + * Lock to make sure another thread attempting preemption doesn't skip it 196 + * while we are still evaluating the next ring. This makes sure the other 197 + * thread does start preemption if we abort it and avoids a soft lock. 198 + */ 199 + spin_lock_irqsave(&a6xx_gpu->eval_lock, flags); 200 + 201 + /* 202 + * Try to start preemption by moving from NONE to START. If 203 + * unsuccessful, a preemption is already in flight 204 + */ 205 + if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START)) { 206 + spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 207 + return; 208 + } 209 + 210 + cntl = A6XX_CP_CONTEXT_SWITCH_CNTL_LEVEL(a6xx_gpu->preempt_level); 211 + 212 + if (a6xx_gpu->skip_save_restore) 213 + cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_SKIP_SAVE_RESTORE; 214 + 215 + if (a6xx_gpu->uses_gmem) 216 + cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_USES_GMEM; 217 + 218 + cntl |= A6XX_CP_CONTEXT_SWITCH_CNTL_STOP; 219 + 220 + /* Get the next ring to preempt to */ 221 + ring = get_next_ring(gpu); 222 + 223 + /* 224 + * If no ring is populated or the highest priority ring is the current 225 + * one do nothing except to update the wptr to the latest and greatest 226 + */ 227 + if (!ring || (a6xx_gpu->cur_ring == ring)) { 228 + set_preempt_state(a6xx_gpu, PREEMPT_FINISH); 229 + update_wptr(gpu, a6xx_gpu->cur_ring); 230 + set_preempt_state(a6xx_gpu, PREEMPT_NONE); 231 + spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 232 + return; 233 + } 234 + 235 + spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags); 236 + 237 + spin_lock_irqsave(&ring->preempt_lock, flags); 238 + 239 + struct a7xx_cp_smmu_info *smmu_info_ptr = 240 + a6xx_gpu->preempt_smmu[ring->id]; 241 + struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[ring->id]; 242 + u64 ttbr0 = ring->memptrs->ttbr0; 243 + u32 context_idr = ring->memptrs->context_idr; 244 + 245 + smmu_info_ptr->ttbr0 = ttbr0; 246 + smmu_info_ptr->context_idr = context_idr; 247 + record_ptr->wptr = get_wptr(ring); 248 + 249 + /* 250 + * The GPU will write the wptr we set above when we preempt. Reset 251 + * restore_wptr to make sure that we don't write WPTR to the same 252 + * thing twice. It's still possible subsequent submissions will update 253 + * wptr again, in which case they will set the flag to true. This has 254 + * to be protected by the lock for setting the flag and updating wptr 255 + * to be atomic. 256 + */ 257 + ring->restore_wptr = false; 258 + 259 + spin_unlock_irqrestore(&ring->preempt_lock, flags); 260 + 261 + gpu_write64(gpu, 262 + REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO, 263 + a6xx_gpu->preempt_smmu_iova[ring->id]); 264 + 265 + gpu_write64(gpu, 266 + REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR, 267 + a6xx_gpu->preempt_iova[ring->id]); 268 + 269 + a6xx_gpu->next_ring = ring; 270 + 271 + /* Start a timer to catch a stuck preemption */ 272 + mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 273 + 274 + /* Set the preemption state to triggered */ 275 + set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED); 276 + 277 + /* Trigger the preemption */ 278 + gpu_write(gpu, REG_A6XX_CP_CONTEXT_SWITCH_CNTL, cntl); 279 + } 280 + 281 + static int preempt_init_ring(struct a6xx_gpu *a6xx_gpu, 282 + struct msm_ringbuffer *ring) 283 + { 284 + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 285 + struct msm_gpu *gpu = &adreno_gpu->base; 286 + struct drm_gem_object *bo = NULL; 287 + phys_addr_t ttbr; 288 + u64 iova = 0; 289 + void *ptr; 290 + int asid; 291 + 292 + ptr = msm_gem_kernel_new(gpu->dev, 293 + PREEMPT_RECORD_SIZE(adreno_gpu), 294 + MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova); 295 + 296 + if (IS_ERR(ptr)) 297 + return PTR_ERR(ptr); 298 + 299 + memset(ptr, 0, PREEMPT_RECORD_SIZE(adreno_gpu)); 300 + 301 + msm_gem_object_set_name(bo, "preempt_record ring%d", ring->id); 302 + 303 + a6xx_gpu->preempt_bo[ring->id] = bo; 304 + a6xx_gpu->preempt_iova[ring->id] = iova; 305 + a6xx_gpu->preempt[ring->id] = ptr; 306 + 307 + struct a6xx_preempt_record *record_ptr = ptr; 308 + 309 + ptr = msm_gem_kernel_new(gpu->dev, 310 + PREEMPT_SMMU_INFO_SIZE, 311 + MSM_BO_WC | MSM_BO_MAP_PRIV | MSM_BO_GPU_READONLY, 312 + gpu->aspace, &bo, &iova); 313 + 314 + if (IS_ERR(ptr)) 315 + return PTR_ERR(ptr); 316 + 317 + memset(ptr, 0, PREEMPT_SMMU_INFO_SIZE); 318 + 319 + msm_gem_object_set_name(bo, "preempt_smmu_info ring%d", ring->id); 320 + 321 + a6xx_gpu->preempt_smmu_bo[ring->id] = bo; 322 + a6xx_gpu->preempt_smmu_iova[ring->id] = iova; 323 + a6xx_gpu->preempt_smmu[ring->id] = ptr; 324 + 325 + struct a7xx_cp_smmu_info *smmu_info_ptr = ptr; 326 + 327 + msm_iommu_pagetable_params(gpu->aspace->mmu, &ttbr, &asid); 328 + 329 + smmu_info_ptr->magic = GEN7_CP_SMMU_INFO_MAGIC; 330 + smmu_info_ptr->ttbr0 = ttbr; 331 + smmu_info_ptr->asid = 0xdecafbad; 332 + smmu_info_ptr->context_idr = 0; 333 + 334 + /* Set up the defaults on the preemption record */ 335 + record_ptr->magic = A6XX_PREEMPT_RECORD_MAGIC; 336 + record_ptr->info = 0; 337 + record_ptr->data = 0; 338 + record_ptr->rptr = 0; 339 + record_ptr->wptr = 0; 340 + record_ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; 341 + record_ptr->rbase = ring->iova; 342 + record_ptr->counter = 0; 343 + record_ptr->bv_rptr_addr = rbmemptr(ring, bv_rptr); 344 + 345 + return 0; 346 + } 347 + 348 + void a6xx_preempt_fini(struct msm_gpu *gpu) 349 + { 350 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 351 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 352 + int i; 353 + 354 + for (i = 0; i < gpu->nr_rings; i++) 355 + msm_gem_kernel_put(a6xx_gpu->preempt_bo[i], gpu->aspace); 356 + } 357 + 358 + void a6xx_preempt_init(struct msm_gpu *gpu) 359 + { 360 + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 361 + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); 362 + int i; 363 + 364 + /* No preemption if we only have one ring */ 365 + if (gpu->nr_rings <= 1) 366 + return; 367 + 368 + for (i = 0; i < gpu->nr_rings; i++) { 369 + if (preempt_init_ring(a6xx_gpu, gpu->rb[i])) 370 + goto fail; 371 + } 372 + 373 + /* TODO: make this configurable? */ 374 + a6xx_gpu->preempt_level = 1; 375 + a6xx_gpu->uses_gmem = 1; 376 + a6xx_gpu->skip_save_restore = 1; 377 + 378 + timer_setup(&a6xx_gpu->preempt_timer, a6xx_preempt_timer, 0); 379 + 380 + return; 381 + fail: 382 + /* 383 + * On any failure our adventure is over. Clean up and 384 + * set nr_rings to 1 to force preemption off 385 + */ 386 + a6xx_preempt_fini(gpu); 387 + gpu->nr_rings = 1; 388 + 389 + DRM_DEV_ERROR(&gpu->pdev->dev, 390 + "preemption init failed, disabling preemption\n"); 391 + 392 + return; 393 + }

+7

drivers/gpu/drm/msm/msm_ringbuffer.h

··· 36 36 37 37 volatile struct msm_gpu_submit_stats stats[MSM_GPU_SUBMIT_STATS_COUNT]; 38 38 volatile u64 ttbr0; 39 + volatile u32 context_idr; 39 40 }; 40 41 41 42 struct msm_cp_state { ··· 101 100 * preemption. Can be aquired from irq context. 102 101 */ 103 102 spinlock_t preempt_lock; 103 + 104 + /* 105 + * Whether we skipped writing wptr and it needs to be updated in the 106 + * future when the ring becomes current. 107 + */ 108 + bool restore_wptr; 104 109 105 110 /** 106 111 * cur_ctx_seqno:

Configure Feed

Configure Feed