Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
drm/msm/a8xx: Preemption support for A840
The programing sequence related to preemption is unchanged from A7x. But
there is some code churn due to register shuffling in A8x. So, split out
the common code into a header file for code sharing and add/update
additional changes required to support preemption feature on A8x GPUs.
Finally, enable the preemption quirk in A840's catalog to enable this
feature.
Signed-off-by: Akhil P Oommen <akhilpo@oss.qualcomm.com>
Patchwork: https://patchwork.freedesktop.org/patch/714682/
Message-ID: <20260327-a8xx-gpu-batch2-v2-15-2b53c38d2101@oss.qualcomm.com>
Signed-off-by: Rob Clark <robin.clark@oss.qualcomm.com>
authored by
Akhil P Oommen
and committed by
Rob Clark
a693602e
ee37487f
+389
-80
+1
drivers/gpu/drm/msm/Makefile
+1
drivers/gpu/drm/msm/Makefile
+1
drivers/gpu/drm/msm/adreno/a6xx_catalog.c
+1
drivers/gpu/drm/msm/adreno/a6xx_catalog.c
+5
-2
drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+5
-2
drivers/gpu/drm/msm/adreno/a6xx_gpu.c
···
410
410
a6xx_flush(gpu, ring);
411
411
}
412
412
413
-
static void a6xx_emit_set_pseudo_reg(struct msm_ringbuffer *ring,
413
+
void a6xx_emit_set_pseudo_reg(struct msm_ringbuffer *ring,
414
414
struct a6xx_gpu *a6xx_gpu, struct msm_gpu_submitqueue *queue)
415
415
{
416
416
u64 preempt_postamble;
···
620
620
a6xx_flush(gpu, ring);
621
621
622
622
/* Check to see if we need to start preemption */
623
-
a6xx_preempt_trigger(gpu);
623
+
if (adreno_is_a8xx(adreno_gpu))
624
+
a8xx_preempt_trigger(gpu);
625
+
else
626
+
a6xx_preempt_trigger(gpu);
624
627
}
625
628
626
629
static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
+5
drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+5
drivers/gpu/drm/msm/adreno/a6xx_gpu.h
···
278
278
void a6xx_preempt_trigger(struct msm_gpu *gpu);
279
279
void a6xx_preempt_irq(struct msm_gpu *gpu);
280
280
void a6xx_preempt_fini(struct msm_gpu *gpu);
281
+
void a6xx_emit_set_pseudo_reg(struct msm_ringbuffer *ring,
282
+
struct a6xx_gpu *a6xx_gpu, struct msm_gpu_submitqueue *queue);
281
283
int a6xx_preempt_submitqueue_setup(struct msm_gpu *gpu,
282
284
struct msm_gpu_submitqueue *queue);
283
285
void a6xx_preempt_submitqueue_close(struct msm_gpu *gpu,
···
329
327
int a8xx_hw_init(struct msm_gpu *gpu);
330
328
irqreturn_t a8xx_irq(struct msm_gpu *gpu);
331
329
void a8xx_llc_activate(struct a6xx_gpu *a6xx_gpu);
330
+
void a8xx_preempt_hw_init(struct msm_gpu *gpu);
331
+
void a8xx_preempt_trigger(struct msm_gpu *gpu);
332
+
void a8xx_preempt_irq(struct msm_gpu *gpu);
332
333
bool a8xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
333
334
void a8xx_recover(struct msm_gpu *gpu);
334
335
#endif /* __A6XX_GPU_H__ */
+1
-76
drivers/gpu/drm/msm/adreno/a6xx_preempt.c
+1
-76
drivers/gpu/drm/msm/adreno/a6xx_preempt.c
···
6
6
#include "msm_gem.h"
7
7
#include "a6xx_gpu.h"
8
8
#include "a6xx_gmu.xml.h"
9
+
#include "a6xx_preempt.h"
9
10
#include "msm_mmu.h"
10
11
#include "msm_gpu_trace.h"
11
-
12
-
/*
13
-
* Try to transition the preemption state from old to new. Return
14
-
* true on success or false if the original state wasn't 'old'
15
-
*/
16
-
static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu,
17
-
enum a6xx_preempt_state old, enum a6xx_preempt_state new)
18
-
{
19
-
enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state,
20
-
old, new);
21
-
22
-
return (cur == old);
23
-
}
24
-
25
-
/*
26
-
* Force the preemption state to the specified state. This is used in cases
27
-
* where the current state is known and won't change
28
-
*/
29
-
static inline void set_preempt_state(struct a6xx_gpu *gpu,
30
-
enum a6xx_preempt_state new)
31
-
{
32
-
/*
33
-
* preempt_state may be read by other cores trying to trigger a
34
-
* preemption or in the interrupt handler so barriers are needed
35
-
* before...
36
-
*/
37
-
smp_mb__before_atomic();
38
-
atomic_set(&gpu->preempt_state, new);
39
-
/* ... and after*/
40
-
smp_mb__after_atomic();
41
-
}
42
-
43
-
/* Write the most recent wptr for the given ring into the hardware */
44
-
static inline void update_wptr(struct a6xx_gpu *a6xx_gpu, struct msm_ringbuffer *ring)
45
-
{
46
-
unsigned long flags;
47
-
uint32_t wptr;
48
-
49
-
spin_lock_irqsave(&ring->preempt_lock, flags);
50
-
51
-
if (ring->restore_wptr) {
52
-
wptr = get_wptr(ring);
53
-
54
-
a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_RB_WPTR, wptr, BIT(0), false);
55
-
56
-
ring->restore_wptr = false;
57
-
}
58
-
59
-
spin_unlock_irqrestore(&ring->preempt_lock, flags);
60
-
}
61
-
62
-
/* Return the highest priority ringbuffer with something in it */
63
-
static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
64
-
{
65
-
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
66
-
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
67
-
68
-
unsigned long flags;
69
-
int i;
70
-
71
-
for (i = 0; i < gpu->nr_rings; i++) {
72
-
bool empty;
73
-
struct msm_ringbuffer *ring = gpu->rb[i];
74
-
75
-
spin_lock_irqsave(&ring->preempt_lock, flags);
76
-
empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
77
-
if (!empty && ring == a6xx_gpu->cur_ring)
78
-
empty = ring->memptrs->fence == a6xx_gpu->last_seqno[i];
79
-
spin_unlock_irqrestore(&ring->preempt_lock, flags);
80
-
81
-
if (!empty)
82
-
return ring;
83
-
}
84
-
85
-
return NULL;
86
-
}
87
12
88
13
static void a6xx_preempt_timer(struct timer_list *t)
89
14
{
+82
drivers/gpu/drm/msm/adreno/a6xx_preempt.h
+82
drivers/gpu/drm/msm/adreno/a6xx_preempt.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/* Copyright (c) 2018, The Linux Foundation. All rights reserved. */
3
+
/* Copyright (c) 2023 Collabora, Ltd. */
4
+
/* Copyright (c) 2024 Valve Corporation */
5
+
/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
6
+
7
+
/*
8
+
* Try to transition the preemption state from old to new. Return
9
+
* true on success or false if the original state wasn't 'old'
10
+
*/
11
+
static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu,
12
+
enum a6xx_preempt_state old, enum a6xx_preempt_state new)
13
+
{
14
+
enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state,
15
+
old, new);
16
+
17
+
return (cur == old);
18
+
}
19
+
20
+
/*
21
+
* Force the preemption state to the specified state. This is used in cases
22
+
* where the current state is known and won't change
23
+
*/
24
+
static inline void set_preempt_state(struct a6xx_gpu *gpu,
25
+
enum a6xx_preempt_state new)
26
+
{
27
+
/*
28
+
* preempt_state may be read by other cores trying to trigger a
29
+
* preemption or in the interrupt handler so barriers are needed
30
+
* before...
31
+
*/
32
+
smp_mb__before_atomic();
33
+
atomic_set(&gpu->preempt_state, new);
34
+
/* ... and after */
35
+
smp_mb__after_atomic();
36
+
}
37
+
38
+
/* Write the most recent wptr for the given ring into the hardware */
39
+
static inline void update_wptr(struct a6xx_gpu *a6xx_gpu, struct msm_ringbuffer *ring)
40
+
{
41
+
unsigned long flags;
42
+
uint32_t wptr;
43
+
44
+
spin_lock_irqsave(&ring->preempt_lock, flags);
45
+
46
+
if (ring->restore_wptr) {
47
+
wptr = get_wptr(ring);
48
+
49
+
a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_RB_WPTR, wptr, BIT(0), false);
50
+
51
+
ring->restore_wptr = false;
52
+
}
53
+
54
+
spin_unlock_irqrestore(&ring->preempt_lock, flags);
55
+
}
56
+
57
+
/* Return the highest priority ringbuffer with something in it */
58
+
static inline struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
59
+
{
60
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
61
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
62
+
63
+
unsigned long flags;
64
+
int i;
65
+
66
+
for (i = 0; i < gpu->nr_rings; i++) {
67
+
bool empty;
68
+
struct msm_ringbuffer *ring = gpu->rb[i];
69
+
70
+
spin_lock_irqsave(&ring->preempt_lock, flags);
71
+
empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
72
+
if (!empty && ring == a6xx_gpu->cur_ring)
73
+
empty = ring->memptrs->fence == a6xx_gpu->last_seqno[i];
74
+
spin_unlock_irqrestore(&ring->preempt_lock, flags);
75
+
76
+
if (!empty)
77
+
return ring;
78
+
}
79
+
80
+
return NULL;
81
+
}
82
+
+35
-2
drivers/gpu/drm/msm/adreno/a8xx_gpu.c
+35
-2
drivers/gpu/drm/msm/adreno/a8xx_gpu.c
···
473
473
a8xx_aperture_clear(gpu);
474
474
}
475
475
476
+
static int a8xx_preempt_start(struct msm_gpu *gpu)
477
+
{
478
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
479
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
480
+
struct msm_ringbuffer *ring = gpu->rb[0];
481
+
482
+
if (gpu->nr_rings <= 1)
483
+
return 0;
484
+
485
+
/* Turn CP protection off */
486
+
OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
487
+
OUT_RING(ring, 0);
488
+
489
+
a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, NULL);
490
+
491
+
/* Yield the floor on command completion */
492
+
OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
493
+
OUT_RING(ring, 0x00);
494
+
OUT_RING(ring, 0x00);
495
+
OUT_RING(ring, 0x00);
496
+
/* Generate interrupt on preemption completion */
497
+
OUT_RING(ring, 0x00);
498
+
499
+
a6xx_flush(gpu, ring);
500
+
501
+
return a8xx_idle(gpu, ring) ? 0 : -EINVAL;
502
+
}
503
+
476
504
static int a8xx_cp_init(struct msm_gpu *gpu)
477
505
{
478
506
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
···
776
748
gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR, shadowptr(a6xx_gpu, gpu->rb[0]));
777
749
gpu_write64(gpu, REG_A8XX_CP_RB_RPTR_ADDR_BV, rbmemptr(gpu->rb[0], bv_rptr));
778
750
751
+
a8xx_preempt_hw_init(gpu);
752
+
779
753
for (i = 0; i < gpu->nr_rings; i++)
780
754
a6xx_gpu->shadow[i] = 0;
781
755
···
840
810
/* Enable hardware clockgating */
841
811
a8xx_set_hwcg(gpu, true);
842
812
out:
813
+
/* Last step - yield the ringbuffer */
814
+
a8xx_preempt_start(gpu);
815
+
843
816
/*
844
817
* Tell the GMU that we are done touching the GPU and it can start power
845
818
* management
···
1252
1219
1253
1220
if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1254
1221
msm_gpu_retire(gpu);
1255
-
a6xx_preempt_trigger(gpu);
1222
+
a8xx_preempt_trigger(gpu);
1256
1223
}
1257
1224
1258
1225
if (status & A6XX_RBBM_INT_0_MASK_CP_SW)
1259
-
a6xx_preempt_irq(gpu);
1226
+
a8xx_preempt_irq(gpu);
1260
1227
1261
1228
return IRQ_HANDLED;
1262
1229
}
+259
drivers/gpu/drm/msm/adreno/a8xx_preempt.c
+259
drivers/gpu/drm/msm/adreno/a8xx_preempt.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
3
+
4
+
#include "msm_gem.h"
5
+
#include "a6xx_gpu.h"
6
+
#include "a6xx_gmu.xml.h"
7
+
#include "a6xx_preempt.h"
8
+
#include "msm_mmu.h"
9
+
#include "msm_gpu_trace.h"
10
+
11
+
static void preempt_prepare_postamble(struct a6xx_gpu *a6xx_gpu)
12
+
{
13
+
u32 *postamble = a6xx_gpu->preempt_postamble_ptr;
14
+
u32 count = 0;
15
+
16
+
postamble[count++] = PKT7(CP_REG_RMW, 3);
17
+
postamble[count++] = REG_A8XX_RBBM_PERFCTR_SRAM_INIT_CMD;
18
+
postamble[count++] = 0;
19
+
postamble[count++] = 1;
20
+
21
+
postamble[count++] = PKT7(CP_WAIT_REG_MEM, 6);
22
+
postamble[count++] = CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ);
23
+
postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_LO(
24
+
REG_A8XX_RBBM_PERFCTR_SRAM_INIT_STATUS);
25
+
postamble[count++] = CP_WAIT_REG_MEM_POLL_ADDR_HI(0);
26
+
postamble[count++] = CP_WAIT_REG_MEM_3_REF(0x1);
27
+
postamble[count++] = CP_WAIT_REG_MEM_4_MASK(0x1);
28
+
postamble[count++] = CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0);
29
+
30
+
a6xx_gpu->preempt_postamble_len = count;
31
+
32
+
a6xx_gpu->postamble_enabled = true;
33
+
}
34
+
35
+
static void preempt_disable_postamble(struct a6xx_gpu *a6xx_gpu)
36
+
{
37
+
u32 *postamble = a6xx_gpu->preempt_postamble_ptr;
38
+
39
+
/*
40
+
* Disable the postamble by replacing the first packet header with a NOP
41
+
* that covers the whole buffer.
42
+
*/
43
+
*postamble = PKT7(CP_NOP, (a6xx_gpu->preempt_postamble_len - 1));
44
+
45
+
a6xx_gpu->postamble_enabled = false;
46
+
}
47
+
48
+
/*
49
+
* Set preemption keepalive vote. Please note that this vote is different from the one used in
50
+
* a8xx_irq()
51
+
*/
52
+
static void a8xx_preempt_keepalive_vote(struct msm_gpu *gpu, bool on)
53
+
{
54
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
55
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
56
+
57
+
gmu_write(&a6xx_gpu->gmu, REG_A8XX_GMU_PWR_COL_PREEMPT_KEEPALIVE, on);
58
+
}
59
+
60
+
void a8xx_preempt_irq(struct msm_gpu *gpu)
61
+
{
62
+
uint32_t status;
63
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
64
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
65
+
struct drm_device *dev = gpu->dev;
66
+
67
+
if (!try_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
68
+
return;
69
+
70
+
/* Delete the preemption watchdog timer */
71
+
timer_delete(&a6xx_gpu->preempt_timer);
72
+
73
+
/*
74
+
* The hardware should be setting the stop bit of CP_CONTEXT_SWITCH_CNTL
75
+
* to zero before firing the interrupt, but there is a non zero chance
76
+
* of a hardware condition or a software race that could set it again
77
+
* before we have a chance to finish. If that happens, log and go for
78
+
* recovery
79
+
*/
80
+
status = gpu_read(gpu, REG_A8XX_CP_CONTEXT_SWITCH_CNTL);
81
+
if (unlikely(status & A8XX_CP_CONTEXT_SWITCH_CNTL_STOP)) {
82
+
DRM_DEV_ERROR(&gpu->pdev->dev,
83
+
"!!!!!!!!!!!!!!!! preemption faulted !!!!!!!!!!!!!! irq\n");
84
+
set_preempt_state(a6xx_gpu, PREEMPT_FAULTED);
85
+
dev_err(dev->dev, "%s: Preemption failed to complete\n",
86
+
gpu->name);
87
+
kthread_queue_work(gpu->worker, &gpu->recover_work);
88
+
return;
89
+
}
90
+
91
+
a6xx_gpu->cur_ring = a6xx_gpu->next_ring;
92
+
a6xx_gpu->next_ring = NULL;
93
+
94
+
set_preempt_state(a6xx_gpu, PREEMPT_FINISH);
95
+
96
+
update_wptr(a6xx_gpu, a6xx_gpu->cur_ring);
97
+
98
+
set_preempt_state(a6xx_gpu, PREEMPT_NONE);
99
+
100
+
a8xx_preempt_keepalive_vote(gpu, false);
101
+
102
+
trace_msm_gpu_preemption_irq(a6xx_gpu->cur_ring->id);
103
+
104
+
/*
105
+
* Retrigger preemption to avoid a deadlock that might occur when preemption
106
+
* is skipped due to it being already in flight when requested.
107
+
*/
108
+
a8xx_preempt_trigger(gpu);
109
+
}
110
+
111
+
void a8xx_preempt_hw_init(struct msm_gpu *gpu)
112
+
{
113
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
114
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
115
+
int i;
116
+
117
+
/* No preemption if we only have one ring */
118
+
if (gpu->nr_rings == 1)
119
+
return;
120
+
121
+
for (i = 0; i < gpu->nr_rings; i++) {
122
+
struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[i];
123
+
124
+
record_ptr->wptr = 0;
125
+
record_ptr->rptr = 0;
126
+
record_ptr->rptr_addr = shadowptr(a6xx_gpu, gpu->rb[i]);
127
+
record_ptr->info = 0;
128
+
record_ptr->data = 0;
129
+
record_ptr->rbase = gpu->rb[i]->iova;
130
+
}
131
+
132
+
/* Write a 0 to signal that we aren't switching pagetables */
133
+
gpu_write64(gpu, REG_A8XX_CP_CONTEXT_SWITCH_SMMU_INFO, 0);
134
+
135
+
/* Enable the GMEM save/restore feature for preemption */
136
+
gpu_write(gpu, REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE, 0x1);
137
+
138
+
/* Reset the preemption state */
139
+
set_preempt_state(a6xx_gpu, PREEMPT_NONE);
140
+
141
+
spin_lock_init(&a6xx_gpu->eval_lock);
142
+
143
+
/* Always come up on rb 0 */
144
+
a6xx_gpu->cur_ring = gpu->rb[0];
145
+
}
146
+
147
+
void a8xx_preempt_trigger(struct msm_gpu *gpu)
148
+
{
149
+
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
150
+
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
151
+
unsigned long flags;
152
+
struct msm_ringbuffer *ring;
153
+
unsigned int cntl;
154
+
bool sysprof;
155
+
156
+
if (gpu->nr_rings == 1)
157
+
return;
158
+
159
+
/*
160
+
* Lock to make sure another thread attempting preemption doesn't skip it
161
+
* while we are still evaluating the next ring. This makes sure the other
162
+
* thread does start preemption if we abort it and avoids a soft lock.
163
+
*/
164
+
spin_lock_irqsave(&a6xx_gpu->eval_lock, flags);
165
+
166
+
/*
167
+
* Try to start preemption by moving from NONE to START. If
168
+
* unsuccessful, a preemption is already in flight
169
+
*/
170
+
if (!try_preempt_state(a6xx_gpu, PREEMPT_NONE, PREEMPT_START)) {
171
+
spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags);
172
+
return;
173
+
}
174
+
175
+
cntl = A8XX_CP_CONTEXT_SWITCH_CNTL_LEVEL(a6xx_gpu->preempt_level);
176
+
177
+
if (a6xx_gpu->skip_save_restore)
178
+
cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_SKIP_SAVE_RESTORE;
179
+
180
+
if (a6xx_gpu->uses_gmem)
181
+
cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_USES_GMEM;
182
+
183
+
cntl |= A8XX_CP_CONTEXT_SWITCH_CNTL_STOP;
184
+
185
+
/* Get the next ring to preempt to */
186
+
ring = get_next_ring(gpu);
187
+
188
+
/*
189
+
* If no ring is populated or the highest priority ring is the current
190
+
* one do nothing except to update the wptr to the latest and greatest
191
+
*/
192
+
if (!ring || (a6xx_gpu->cur_ring == ring)) {
193
+
set_preempt_state(a6xx_gpu, PREEMPT_FINISH);
194
+
update_wptr(a6xx_gpu, a6xx_gpu->cur_ring);
195
+
set_preempt_state(a6xx_gpu, PREEMPT_NONE);
196
+
spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags);
197
+
return;
198
+
}
199
+
200
+
spin_unlock_irqrestore(&a6xx_gpu->eval_lock, flags);
201
+
202
+
spin_lock_irqsave(&ring->preempt_lock, flags);
203
+
204
+
struct a7xx_cp_smmu_info *smmu_info_ptr =
205
+
a6xx_gpu->preempt_smmu[ring->id];
206
+
struct a6xx_preempt_record *record_ptr = a6xx_gpu->preempt[ring->id];
207
+
u64 ttbr0 = ring->memptrs->ttbr0;
208
+
u32 context_idr = ring->memptrs->context_idr;
209
+
210
+
smmu_info_ptr->ttbr0 = ttbr0;
211
+
smmu_info_ptr->context_idr = context_idr;
212
+
record_ptr->wptr = get_wptr(ring);
213
+
214
+
/*
215
+
* The GPU will write the wptr we set above when we preempt. Reset
216
+
* restore_wptr to make sure that we don't write WPTR to the same
217
+
* thing twice. It's still possible subsequent submissions will update
218
+
* wptr again, in which case they will set the flag to true. This has
219
+
* to be protected by the lock for setting the flag and updating wptr
220
+
* to be atomic.
221
+
*/
222
+
ring->restore_wptr = false;
223
+
224
+
trace_msm_gpu_preemption_trigger(a6xx_gpu->cur_ring->id, ring->id);
225
+
226
+
spin_unlock_irqrestore(&ring->preempt_lock, flags);
227
+
228
+
/* Set the keepalive bit to keep the GPU ON until preemption is complete */
229
+
a8xx_preempt_keepalive_vote(gpu, true);
230
+
231
+
a6xx_fenced_write(a6xx_gpu,
232
+
REG_A8XX_CP_CONTEXT_SWITCH_SMMU_INFO, a6xx_gpu->preempt_smmu_iova[ring->id],
233
+
BIT(1), true);
234
+
235
+
a6xx_fenced_write(a6xx_gpu,
236
+
REG_A8XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR,
237
+
a6xx_gpu->preempt_iova[ring->id], BIT(1), true);
238
+
239
+
a6xx_gpu->next_ring = ring;
240
+
241
+
/* Start a timer to catch a stuck preemption */
242
+
mod_timer(&a6xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
243
+
244
+
/* Enable or disable postamble as needed */
245
+
sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1;
246
+
247
+
if (!sysprof && !a6xx_gpu->postamble_enabled)
248
+
preempt_prepare_postamble(a6xx_gpu);
249
+
250
+
if (sysprof && a6xx_gpu->postamble_enabled)
251
+
preempt_disable_postamble(a6xx_gpu);
252
+
253
+
/* Set the preemption state to triggered */
254
+
set_preempt_state(a6xx_gpu, PREEMPT_TRIGGERED);
255
+
256
+
/* Trigger the preemption */
257
+
a6xx_fenced_write(a6xx_gpu, REG_A8XX_CP_CONTEXT_SWITCH_CNTL, cntl, BIT(1), false);
258
+
}
259
+