Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
drm/amdgpu: Add initial support for gfx v12_1
Add initial support for gfx ip block for gc v12_1.
V2: Remove some not applicable bit set.
V3: drop unused header (Alex)
v4: Fix type on copyright
v5: fix num_xcc handling (Alex)
Signed-off-by: Likun Gao <Likun.Gao@amd.com>
Reviewed-by: Hawking Zhang <Hawking.Zhang@amd.com>
Reviewed-by: Mukul Joshi <mukul.joshi@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
authored by
Likun Gao
and committed by
Alex Deucher
ad5f1ee0
78996e1e
+3978
+1
drivers/gpu/drm/amd/amdgpu/Makefile
+1
drivers/gpu/drm/amd/amdgpu/Makefile
+3948
drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+3948
drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
···
1
+
/*
2
+
* Copyright 2025 Advanced Micro Devices, Inc.
3
+
*
4
+
* Permission is hereby granted, free of charge, to any person obtaining a
5
+
* copy of this software and associated documentation files (the "Software"),
6
+
* to deal in the Software without restriction, including without limitation
7
+
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
+
* and/or sell copies of the Software, and to permit persons to whom the
9
+
* Software is furnished to do so, subject to the following conditions:
10
+
*
11
+
* The above copyright notice and this permission notice shall be included in
12
+
* all copies or substantial portions of the Software.
13
+
*
14
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17
+
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
+
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
+
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
+
* OTHER DEALINGS IN THE SOFTWARE.
21
+
*
22
+
*/
23
+
#include <linux/delay.h>
24
+
#include <linux/kernel.h>
25
+
#include <linux/firmware.h>
26
+
#include <linux/module.h>
27
+
#include <linux/pci.h>
28
+
#include "amdgpu.h"
29
+
#include "amdgpu_gfx.h"
30
+
#include "amdgpu_psp.h"
31
+
#include "amdgpu_smu.h"
32
+
#include "amdgpu_atomfirmware.h"
33
+
#include "imu_v12_0.h"
34
+
#include "soc_v1_0.h"
35
+
#include "gfx_v12_1_pkt.h"
36
+
37
+
#include "gc/gc_12_1_0_offset.h"
38
+
#include "gc/gc_12_1_0_sh_mask.h"
39
+
#include "soc24_enum.h"
40
+
#include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
41
+
42
+
#include "soc15.h"
43
+
#include "soc15d.h"
44
+
#include "clearstate_gfx12.h"
45
+
#include "v12_structs.h"
46
+
#include "gfx_v12_1.h"
47
+
#include "mes_v12_1.h"
48
+
49
+
#define GFX12_MEC_HPD_SIZE 2048
50
+
51
+
#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
52
+
53
+
MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
54
+
MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin");
55
+
56
+
#define DEFAULT_SH_MEM_CONFIG \
57
+
((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
58
+
(SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
59
+
(3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
60
+
61
+
static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id);
62
+
static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev);
63
+
static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev);
64
+
static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev);
65
+
static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev);
66
+
static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev);
67
+
static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
68
+
struct amdgpu_cu_info *cu_info);
69
+
static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev);
70
+
static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
71
+
u32 sh_num, u32 instance, int xcc_id);
72
+
static u32 gfx_v12_1_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev,
73
+
int xcc_id);
74
+
75
+
static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
76
+
uint32_t val);
77
+
static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
78
+
static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
79
+
uint16_t pasid, uint32_t flush_type,
80
+
bool all_hub, uint8_t dst_sel);
81
+
static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
82
+
static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
83
+
static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
84
+
bool enable);
85
+
static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
86
+
bool enable, int xcc_id);
87
+
88
+
static void gfx_v12_1_kiq_set_resources(struct amdgpu_ring *kiq_ring,
89
+
uint64_t queue_mask)
90
+
{
91
+
amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
92
+
amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
93
+
PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
94
+
amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
95
+
amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
96
+
amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
97
+
amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
98
+
amdgpu_ring_write(kiq_ring, 0); /* oac mask */
99
+
amdgpu_ring_write(kiq_ring, 0);
100
+
}
101
+
102
+
static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring,
103
+
struct amdgpu_ring *ring)
104
+
{
105
+
uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
106
+
uint64_t wptr_addr = ring->wptr_gpu_addr;
107
+
uint32_t me = 0, eng_sel = 0;
108
+
109
+
switch (ring->funcs->type) {
110
+
case AMDGPU_RING_TYPE_COMPUTE:
111
+
me = 1;
112
+
eng_sel = 0;
113
+
break;
114
+
case AMDGPU_RING_TYPE_MES:
115
+
me = 2;
116
+
eng_sel = 5;
117
+
break;
118
+
default:
119
+
WARN_ON(1);
120
+
}
121
+
122
+
amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
123
+
/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
124
+
amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
125
+
PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
126
+
PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
127
+
PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
128
+
PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
129
+
PACKET3_MAP_QUEUES_ME((me)) |
130
+
PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
131
+
PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
132
+
PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
133
+
PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
134
+
amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
135
+
amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
136
+
amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
137
+
amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
138
+
amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
139
+
}
140
+
141
+
static void gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
142
+
struct amdgpu_ring *ring,
143
+
enum amdgpu_unmap_queues_action action,
144
+
u64 gpu_addr, u64 seq)
145
+
{
146
+
struct amdgpu_device *adev = kiq_ring->adev;
147
+
uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
148
+
149
+
if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
150
+
amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr,
151
+
seq, kiq_ring->xcc_id);
152
+
return;
153
+
}
154
+
155
+
amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
156
+
amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
157
+
PACKET3_UNMAP_QUEUES_ACTION(action) |
158
+
PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
159
+
PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
160
+
PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
161
+
amdgpu_ring_write(kiq_ring,
162
+
PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
163
+
164
+
if (action == PREEMPT_QUEUES_NO_UNMAP) {
165
+
amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
166
+
amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
167
+
amdgpu_ring_write(kiq_ring, seq);
168
+
} else {
169
+
amdgpu_ring_write(kiq_ring, 0);
170
+
amdgpu_ring_write(kiq_ring, 0);
171
+
amdgpu_ring_write(kiq_ring, 0);
172
+
}
173
+
}
174
+
175
+
static void gfx_v12_1_kiq_query_status(struct amdgpu_ring *kiq_ring,
176
+
struct amdgpu_ring *ring,
177
+
u64 addr, u64 seq)
178
+
{
179
+
uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
180
+
181
+
amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
182
+
amdgpu_ring_write(kiq_ring,
183
+
PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
184
+
PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
185
+
PACKET3_QUERY_STATUS_COMMAND(2));
186
+
amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
187
+
PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
188
+
PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
189
+
amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
190
+
amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
191
+
amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
192
+
amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
193
+
}
194
+
195
+
static void gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
196
+
uint16_t pasid,
197
+
uint32_t flush_type,
198
+
bool all_hub)
199
+
{
200
+
gfx_v12_1_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
201
+
}
202
+
203
+
static const struct kiq_pm4_funcs gfx_v12_1_kiq_pm4_funcs = {
204
+
.kiq_set_resources = gfx_v12_1_kiq_set_resources,
205
+
.kiq_map_queues = gfx_v12_1_kiq_map_queues,
206
+
.kiq_unmap_queues = gfx_v12_1_kiq_unmap_queues,
207
+
.kiq_query_status = gfx_v12_1_kiq_query_status,
208
+
.kiq_invalidate_tlbs = gfx_v12_1_kiq_invalidate_tlbs,
209
+
.set_resources_size = 8,
210
+
.map_queues_size = 7,
211
+
.unmap_queues_size = 6,
212
+
.query_status_size = 7,
213
+
.invalidate_tlbs_size = 2,
214
+
};
215
+
216
+
static void gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device *adev)
217
+
{
218
+
int i, num_xcc;
219
+
220
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
221
+
for (i =0; i < num_xcc; i++)
222
+
adev->gfx.kiq[i].pmf = &gfx_v12_1_kiq_pm4_funcs;
223
+
}
224
+
225
+
static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
226
+
int mem_space, int opt, uint32_t addr0,
227
+
uint32_t addr1, uint32_t ref,
228
+
uint32_t mask, uint32_t inv)
229
+
{
230
+
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
231
+
amdgpu_ring_write(ring,
232
+
/* memory (1) or register (0) */
233
+
(WAIT_REG_MEM_MEM_SPACE(mem_space) |
234
+
WAIT_REG_MEM_OPERATION(opt) | /* wait */
235
+
WAIT_REG_MEM_FUNCTION(3) | /* equal */
236
+
WAIT_REG_MEM_ENGINE(eng_sel)));
237
+
238
+
if (mem_space)
239
+
BUG_ON(addr0 & 0x3); /* Dword align */
240
+
amdgpu_ring_write(ring, addr0);
241
+
amdgpu_ring_write(ring, addr1);
242
+
amdgpu_ring_write(ring, ref);
243
+
amdgpu_ring_write(ring, mask);
244
+
amdgpu_ring_write(ring, inv); /* poll interval */
245
+
}
246
+
247
+
static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring)
248
+
{
249
+
struct amdgpu_device *adev = ring->adev;
250
+
uint32_t scratch_reg0_offset, xcc_offset;
251
+
uint32_t tmp = 0;
252
+
unsigned i;
253
+
int r;
254
+
255
+
/* Use register offset which is local to XCC in the packet */
256
+
xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
257
+
scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
258
+
WREG32(scratch_reg0_offset, 0xCAFEDEAD);
259
+
tmp = RREG32(scratch_reg0_offset);
260
+
261
+
r = amdgpu_ring_alloc(ring, 5);
262
+
if (r) {
263
+
dev_err(adev->dev,
264
+
"amdgpu: cp failed to lock ring %d (%d).\n",
265
+
ring->idx, r);
266
+
return r;
267
+
}
268
+
269
+
if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
270
+
gfx_v12_1_ring_emit_wreg(ring, xcc_offset, 0xDEADBEEF);
271
+
} else {
272
+
amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
273
+
amdgpu_ring_write(ring, xcc_offset -
274
+
PACKET3_SET_UCONFIG_REG_START);
275
+
amdgpu_ring_write(ring, 0xDEADBEEF);
276
+
}
277
+
amdgpu_ring_commit(ring);
278
+
279
+
for (i = 0; i < adev->usec_timeout; i++) {
280
+
tmp = RREG32(scratch_reg0_offset);
281
+
if (tmp == 0xDEADBEEF)
282
+
break;
283
+
if (amdgpu_emu_mode == 1)
284
+
msleep(1);
285
+
else
286
+
udelay(1);
287
+
}
288
+
289
+
if (i >= adev->usec_timeout)
290
+
r = -ETIMEDOUT;
291
+
return r;
292
+
}
293
+
294
+
static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
295
+
{
296
+
struct amdgpu_device *adev = ring->adev;
297
+
struct amdgpu_ib ib;
298
+
struct dma_fence *f = NULL;
299
+
unsigned index;
300
+
uint64_t gpu_addr;
301
+
volatile uint32_t *cpu_ptr;
302
+
long r;
303
+
304
+
/* MES KIQ fw hasn't indirect buffer support for now */
305
+
if (adev->enable_mes_kiq &&
306
+
ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
307
+
return 0;
308
+
309
+
memset(&ib, 0, sizeof(ib));
310
+
311
+
r = amdgpu_device_wb_get(adev, &index);
312
+
if (r)
313
+
return r;
314
+
315
+
gpu_addr = adev->wb.gpu_addr + (index * 4);
316
+
adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
317
+
cpu_ptr = &adev->wb.wb[index];
318
+
319
+
r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
320
+
if (r) {
321
+
dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
322
+
goto err1;
323
+
}
324
+
325
+
ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
326
+
ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
327
+
ib.ptr[2] = lower_32_bits(gpu_addr);
328
+
ib.ptr[3] = upper_32_bits(gpu_addr);
329
+
ib.ptr[4] = 0xDEADBEEF;
330
+
ib.length_dw = 5;
331
+
332
+
r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
333
+
if (r)
334
+
goto err2;
335
+
336
+
r = dma_fence_wait_timeout(f, false, timeout);
337
+
if (r == 0) {
338
+
r = -ETIMEDOUT;
339
+
goto err2;
340
+
} else if (r < 0) {
341
+
goto err2;
342
+
}
343
+
344
+
if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
345
+
r = 0;
346
+
else
347
+
r = -EINVAL;
348
+
err2:
349
+
amdgpu_ib_free(&ib, NULL);
350
+
dma_fence_put(f);
351
+
err1:
352
+
amdgpu_device_wb_free(adev, index);
353
+
return r;
354
+
}
355
+
356
+
static void gfx_v12_1_free_microcode(struct amdgpu_device *adev)
357
+
{
358
+
amdgpu_ucode_release(&adev->gfx.rlc_fw);
359
+
amdgpu_ucode_release(&adev->gfx.mec_fw);
360
+
361
+
kfree(adev->gfx.rlc.register_list_format);
362
+
}
363
+
364
+
static int gfx_v12_1_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
365
+
{
366
+
const struct psp_firmware_header_v1_0 *toc_hdr;
367
+
int err = 0;
368
+
369
+
err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
370
+
AMDGPU_UCODE_REQUIRED,
371
+
"amdgpu/%s_toc.bin", ucode_prefix);
372
+
if (err)
373
+
goto out;
374
+
375
+
toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
376
+
adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
377
+
adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
378
+
adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
379
+
adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
380
+
le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
381
+
return 0;
382
+
out:
383
+
amdgpu_ucode_release(&adev->psp.toc_fw);
384
+
return err;
385
+
}
386
+
387
+
static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
388
+
{
389
+
char ucode_prefix[15];
390
+
int err;
391
+
const struct rlc_firmware_header_v2_0 *rlc_hdr;
392
+
uint16_t version_major;
393
+
uint16_t version_minor;
394
+
395
+
DRM_DEBUG("\n");
396
+
397
+
amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
398
+
399
+
if (!amdgpu_sriov_vf(adev)) {
400
+
err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
401
+
AMDGPU_UCODE_REQUIRED,
402
+
"amdgpu/%s_rlc.bin", ucode_prefix);
403
+
if (err)
404
+
goto out;
405
+
rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
406
+
version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
407
+
version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
408
+
err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
409
+
if (err)
410
+
goto out;
411
+
}
412
+
413
+
err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
414
+
AMDGPU_UCODE_REQUIRED,
415
+
"amdgpu/%s_mec.bin", ucode_prefix);
416
+
if (err)
417
+
goto out;
418
+
amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
419
+
amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
420
+
amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
421
+
422
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
423
+
err = gfx_v12_1_init_toc_microcode(adev, ucode_prefix);
424
+
425
+
/* only one MEC for gfx 12 */
426
+
adev->gfx.mec2_fw = NULL;
427
+
428
+
if (adev->gfx.imu.funcs) {
429
+
if (adev->gfx.imu.funcs->init_microcode) {
430
+
err = adev->gfx.imu.funcs->init_microcode(adev);
431
+
if (err)
432
+
dev_err(adev->dev, "Failed to load imu firmware!\n");
433
+
}
434
+
}
435
+
436
+
out:
437
+
if (err) {
438
+
amdgpu_ucode_release(&adev->gfx.rlc_fw);
439
+
amdgpu_ucode_release(&adev->gfx.mec_fw);
440
+
}
441
+
442
+
return err;
443
+
}
444
+
445
+
static u32 gfx_v12_1_get_csb_size(struct amdgpu_device *adev)
446
+
{
447
+
u32 count = 0;
448
+
const struct cs_section_def *sect = NULL;
449
+
const struct cs_extent_def *ext = NULL;
450
+
451
+
count += 1;
452
+
453
+
for (sect = gfx12_cs_data; sect->section != NULL; ++sect) {
454
+
if (sect->id == SECT_CONTEXT) {
455
+
for (ext = sect->section; ext->extent != NULL; ++ext)
456
+
count += 2 + ext->reg_count;
457
+
} else
458
+
return 0;
459
+
}
460
+
461
+
return count;
462
+
}
463
+
464
+
static void gfx_v12_1_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
465
+
{
466
+
u32 count = 0, clustercount = 0, i;
467
+
const struct cs_section_def *sect = NULL;
468
+
const struct cs_extent_def *ext = NULL;
469
+
470
+
if (adev->gfx.rlc.cs_data == NULL)
471
+
return;
472
+
if (buffer == NULL)
473
+
return;
474
+
475
+
count += 1;
476
+
477
+
for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
478
+
if (sect->id == SECT_CONTEXT) {
479
+
for (ext = sect->section; ext->extent != NULL; ++ext) {
480
+
clustercount++;
481
+
buffer[count++] = ext->reg_count;
482
+
buffer[count++] = ext->reg_index;
483
+
484
+
for (i = 0; i < ext->reg_count; i++)
485
+
buffer[count++] = cpu_to_le32(ext->extent[i]);
486
+
}
487
+
} else
488
+
return;
489
+
}
490
+
491
+
buffer[0] = clustercount;
492
+
}
493
+
494
+
static void gfx_v12_1_rlc_fini(struct amdgpu_device *adev)
495
+
{
496
+
/* clear state block */
497
+
amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
498
+
&adev->gfx.rlc.clear_state_gpu_addr,
499
+
(void **)&adev->gfx.rlc.cs_ptr);
500
+
501
+
/* jump table block */
502
+
amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
503
+
&adev->gfx.rlc.cp_table_gpu_addr,
504
+
(void **)&adev->gfx.rlc.cp_table_ptr);
505
+
}
506
+
507
+
static void gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
508
+
{
509
+
int xcc_id, num_xcc;
510
+
struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
511
+
512
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
513
+
for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
514
+
reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)];
515
+
reg_access_ctrl->scratch_reg0 =
516
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG0);
517
+
reg_access_ctrl->scratch_reg1 =
518
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG1);
519
+
reg_access_ctrl->scratch_reg2 =
520
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG2);
521
+
reg_access_ctrl->scratch_reg3 =
522
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG3);
523
+
reg_access_ctrl->grbm_cntl =
524
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL);
525
+
reg_access_ctrl->grbm_idx =
526
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
527
+
reg_access_ctrl->spare_int =
528
+
SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT_0);
529
+
}
530
+
adev->gfx.rlc.rlcg_reg_access_supported = true;
531
+
}
532
+
533
+
static int gfx_v12_1_rlc_init(struct amdgpu_device *adev)
534
+
{
535
+
const struct cs_section_def *cs_data;
536
+
int r, i, num_xcc;
537
+
538
+
adev->gfx.rlc.cs_data = gfx12_cs_data;
539
+
540
+
cs_data = adev->gfx.rlc.cs_data;
541
+
542
+
if (cs_data) {
543
+
/* init clear state block */
544
+
r = amdgpu_gfx_rlc_init_csb(adev);
545
+
if (r)
546
+
return r;
547
+
}
548
+
549
+
/* init spm vmid with 0xf */
550
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
551
+
for (i = 0; i < num_xcc; i++) {
552
+
if (adev->gfx.rlc.funcs->update_spm_vmid)
553
+
adev->gfx.rlc.funcs->update_spm_vmid(adev, i, NULL, 0xf);
554
+
}
555
+
556
+
return 0;
557
+
}
558
+
559
+
static void gfx_v12_1_mec_fini(struct amdgpu_device *adev)
560
+
{
561
+
amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
562
+
amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
563
+
amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
564
+
}
565
+
566
+
static int gfx_v12_1_mec_init(struct amdgpu_device *adev)
567
+
{
568
+
int r, i, num_xcc;
569
+
u32 *hpd;
570
+
size_t mec_hpd_size;
571
+
572
+
bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
573
+
574
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
575
+
for (i = 0; i < num_xcc; i++)
576
+
bitmap_zero(adev->gfx.mec_bitmap[i].queue_bitmap,
577
+
AMDGPU_MAX_COMPUTE_QUEUES);
578
+
579
+
/* take ownership of the relevant compute queues */
580
+
amdgpu_gfx_compute_queue_acquire(adev);
581
+
mec_hpd_size = adev->gfx.num_compute_rings *
582
+
GFX12_MEC_HPD_SIZE * num_xcc;
583
+
584
+
if (mec_hpd_size) {
585
+
r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
586
+
AMDGPU_GEM_DOMAIN_GTT,
587
+
&adev->gfx.mec.hpd_eop_obj,
588
+
&adev->gfx.mec.hpd_eop_gpu_addr,
589
+
(void **)&hpd);
590
+
if (r) {
591
+
dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
592
+
gfx_v12_1_mec_fini(adev);
593
+
return r;
594
+
}
595
+
596
+
memset(hpd, 0, mec_hpd_size);
597
+
598
+
amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
599
+
amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
600
+
}
601
+
602
+
return 0;
603
+
}
604
+
605
+
static uint32_t wave_read_ind(struct amdgpu_device *adev,
606
+
uint32_t xcc_id, uint32_t wave,
607
+
uint32_t address)
608
+
{
609
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
610
+
(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
611
+
(address << SQ_IND_INDEX__INDEX__SHIFT));
612
+
return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
613
+
}
614
+
615
+
static void wave_read_regs(struct amdgpu_device *adev,
616
+
uint32_t xcc_id, uint32_t wave,
617
+
uint32_t thread, uint32_t regno,
618
+
uint32_t num, uint32_t *out)
619
+
{
620
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
621
+
(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
622
+
(regno << SQ_IND_INDEX__INDEX__SHIFT) |
623
+
(thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
624
+
(SQ_IND_INDEX__AUTO_INCR_MASK));
625
+
while (num--)
626
+
*(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
627
+
}
628
+
629
+
static void gfx_v12_1_read_wave_data(struct amdgpu_device *adev,
630
+
uint32_t xcc_id,
631
+
uint32_t simd, uint32_t wave,
632
+
uint32_t *dst, int *no_fields)
633
+
{
634
+
/* in gfx12 the SIMD_ID is specified as part of the INSTANCE
635
+
* field when performing a select_se_sh so it should be
636
+
* zero here */
637
+
WARN_ON(simd != 0);
638
+
639
+
/* type 4 wave data */
640
+
dst[(*no_fields)++] = 4;
641
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATUS);
642
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_LO);
643
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_HI);
644
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_LO);
645
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_HI);
646
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID1);
647
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID2);
648
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_GPR_ALLOC);
649
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_LDS_ALLOC);
650
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS);
651
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS2);
652
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_DBG1);
653
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_M0);
654
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_MODE);
655
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATE_PRIV);
656
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
657
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_USER);
658
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_TRAP_CTRL);
659
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_ACTIVE);
660
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_VALID_AND_IDLE);
661
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
662
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
663
+
dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_SCHED_MODE);
664
+
}
665
+
666
+
static void gfx_v12_1_read_wave_sgprs(struct amdgpu_device *adev,
667
+
uint32_t xcc_id, uint32_t simd,
668
+
uint32_t wave, uint32_t start,
669
+
uint32_t size, uint32_t *dst)
670
+
{
671
+
WARN_ON(simd != 0);
672
+
673
+
wave_read_regs(adev, xcc_id, wave, 0,
674
+
start + SQIND_WAVE_SGPRS_OFFSET,
675
+
size, dst);
676
+
}
677
+
678
+
static void gfx_v12_1_read_wave_vgprs(struct amdgpu_device *adev,
679
+
uint32_t xcc_id, uint32_t simd,
680
+
uint32_t wave, uint32_t thread,
681
+
uint32_t start, uint32_t size,
682
+
uint32_t *dst)
683
+
{
684
+
wave_read_regs(adev, xcc_id, wave, thread,
685
+
start + SQIND_WAVE_VGPRS_OFFSET,
686
+
size, dst);
687
+
}
688
+
689
+
static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
690
+
u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
691
+
{
692
+
soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
693
+
}
694
+
695
+
static const struct amdgpu_gfx_funcs gfx_v12_1_gfx_funcs = {
696
+
.get_gpu_clock_counter = &gfx_v12_1_get_gpu_clock_counter,
697
+
.select_se_sh = &gfx_v12_1_xcc_select_se_sh,
698
+
.read_wave_data = &gfx_v12_1_read_wave_data,
699
+
.read_wave_sgprs = &gfx_v12_1_read_wave_sgprs,
700
+
.read_wave_vgprs = &gfx_v12_1_read_wave_vgprs,
701
+
.select_me_pipe_q = &gfx_v12_1_select_me_pipe_q,
702
+
.update_perfmon_mgcg = &gfx_v12_1_update_perf_clk,
703
+
};
704
+
705
+
static int gfx_v12_1_gpu_early_init(struct amdgpu_device *adev)
706
+
{
707
+
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
708
+
case IP_VERSION(12, 1, 0):
709
+
adev->gfx.config.max_hw_contexts = 8;
710
+
adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
711
+
adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
712
+
adev->gfx.config.sc_hiz_tile_fifo_size = 0;
713
+
adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
714
+
break;
715
+
default:
716
+
BUG();
717
+
break;
718
+
}
719
+
720
+
return 0;
721
+
}
722
+
723
+
static int gfx_v12_1_compute_ring_init(struct amdgpu_device *adev, int ring_id,
724
+
int xcc_id, int mec, int pipe, int queue)
725
+
{
726
+
int r;
727
+
unsigned irq_type;
728
+
struct amdgpu_ring *ring;
729
+
unsigned int hw_prio;
730
+
uint32_t xcc_doorbell_start;
731
+
732
+
ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings +
733
+
ring_id];
734
+
735
+
/* mec0 is me1 */
736
+
ring->xcc_id = xcc_id;
737
+
ring->me = mec + 1;
738
+
ring->pipe = pipe;
739
+
ring->queue = queue;
740
+
741
+
ring->ring_obj = NULL;
742
+
ring->use_doorbell = true;
743
+
xcc_doorbell_start = adev->doorbell_index.mec_ring0 +
744
+
xcc_id * adev->doorbell_index.xcc_doorbell_range;
745
+
ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1;
746
+
ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
747
+
(ring_id + xcc_id * adev->gfx.num_compute_rings) *
748
+
GFX12_MEC_HPD_SIZE;
749
+
ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
750
+
sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
751
+
752
+
irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
753
+
+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
754
+
+ ring->pipe;
755
+
hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
756
+
AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
757
+
/* type-2 packets are deprecated on MEC, use type-3 instead */
758
+
r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
759
+
hw_prio, NULL);
760
+
if (r)
761
+
return r;
762
+
763
+
return 0;
764
+
}
765
+
766
+
static struct {
767
+
SOC24_FIRMWARE_ID id;
768
+
unsigned int offset;
769
+
unsigned int size;
770
+
unsigned int size_x16;
771
+
} rlc_autoload_info[SOC24_FIRMWARE_ID_MAX];
772
+
773
+
#define RLC_TOC_OFFSET_DWUNIT 8
774
+
#define RLC_SIZE_MULTIPLE 1024
775
+
#define RLC_TOC_UMF_SIZE_inM 23ULL
776
+
#define RLC_TOC_FORMAT_API 165ULL
777
+
778
+
static void gfx_v12_1_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
779
+
{
780
+
RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc;
781
+
782
+
while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) {
783
+
rlc_autoload_info[ucode->id].id = ucode->id;
784
+
rlc_autoload_info[ucode->id].offset =
785
+
ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4;
786
+
rlc_autoload_info[ucode->id].size =
787
+
ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 :
788
+
ucode->size * 4;
789
+
ucode++;
790
+
}
791
+
}
792
+
793
+
static uint32_t gfx_v12_1_calc_toc_total_size(struct amdgpu_device *adev)
794
+
{
795
+
uint32_t total_size = 0;
796
+
SOC24_FIRMWARE_ID id;
797
+
798
+
gfx_v12_1_parse_rlc_toc(adev, adev->psp.toc.start_addr);
799
+
800
+
for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++)
801
+
total_size += rlc_autoload_info[id].size;
802
+
803
+
/* In case the offset in rlc toc ucode is aligned */
804
+
if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset)
805
+
total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset +
806
+
rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size;
807
+
if (total_size < (RLC_TOC_UMF_SIZE_inM << 20))
808
+
total_size = RLC_TOC_UMF_SIZE_inM << 20;
809
+
810
+
return total_size;
811
+
}
812
+
813
+
static int gfx_v12_1_rlc_autoload_buffer_init(struct amdgpu_device *adev)
814
+
{
815
+
int r;
816
+
uint32_t total_size;
817
+
818
+
total_size = gfx_v12_1_calc_toc_total_size(adev);
819
+
820
+
r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
821
+
AMDGPU_GEM_DOMAIN_VRAM,
822
+
&adev->gfx.rlc.rlc_autoload_bo,
823
+
&adev->gfx.rlc.rlc_autoload_gpu_addr,
824
+
(void **)&adev->gfx.rlc.rlc_autoload_ptr);
825
+
826
+
if (r) {
827
+
dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
828
+
return r;
829
+
}
830
+
831
+
return 0;
832
+
}
833
+
834
+
static void gfx_v12_1_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
835
+
SOC24_FIRMWARE_ID id,
836
+
const void *fw_data,
837
+
uint32_t fw_size)
838
+
{
839
+
uint32_t toc_offset;
840
+
uint32_t toc_fw_size;
841
+
char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
842
+
843
+
if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX)
844
+
return;
845
+
846
+
toc_offset = rlc_autoload_info[id].offset;
847
+
toc_fw_size = rlc_autoload_info[id].size;
848
+
849
+
if (fw_size == 0)
850
+
fw_size = toc_fw_size;
851
+
852
+
if (fw_size > toc_fw_size)
853
+
fw_size = toc_fw_size;
854
+
855
+
memcpy(ptr + toc_offset, fw_data, fw_size);
856
+
857
+
if (fw_size < toc_fw_size)
858
+
memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
859
+
}
860
+
861
+
static void
862
+
gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
863
+
{
864
+
void *data;
865
+
uint32_t size;
866
+
uint32_t *toc_ptr;
867
+
868
+
data = adev->psp.toc.start_addr;
869
+
size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size;
870
+
871
+
toc_ptr = (uint32_t *)data + size / 4 - 2;
872
+
*toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1;
873
+
874
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
875
+
data, size);
876
+
}
877
+
878
+
static void
879
+
gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
880
+
{
881
+
const __le32 *fw_data;
882
+
uint32_t fw_size;
883
+
const struct gfx_firmware_header_v2_0 *cpv2_hdr;
884
+
const struct rlc_firmware_header_v2_0 *rlc_hdr;
885
+
const struct rlc_firmware_header_v2_1 *rlcv21_hdr;
886
+
const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
887
+
uint16_t version_major, version_minor;
888
+
889
+
/* mec ucode */
890
+
cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
891
+
adev->gfx.mec_fw->data;
892
+
/* instruction */
893
+
fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
894
+
le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
895
+
fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
896
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
897
+
fw_data, fw_size);
898
+
/* data */
899
+
fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
900
+
le32_to_cpu(cpv2_hdr->data_offset_bytes));
901
+
fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
902
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
903
+
fw_data, fw_size);
904
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
905
+
fw_data, fw_size);
906
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
907
+
fw_data, fw_size);
908
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
909
+
fw_data, fw_size);
910
+
911
+
/* rlc ucode */
912
+
rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
913
+
adev->gfx.rlc_fw->data;
914
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
915
+
le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
916
+
fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
917
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
918
+
fw_data, fw_size);
919
+
920
+
version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
921
+
version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
922
+
if (version_major == 2) {
923
+
if (version_minor >= 1) {
924
+
rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
925
+
926
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
927
+
le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes));
928
+
fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes);
929
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
930
+
fw_data, fw_size);
931
+
932
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
933
+
le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes));
934
+
fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes);
935
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
936
+
fw_data, fw_size);
937
+
}
938
+
if (version_minor >= 2) {
939
+
rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
940
+
941
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
942
+
le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
943
+
fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
944
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
945
+
fw_data, fw_size);
946
+
947
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
948
+
le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
949
+
fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
950
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
951
+
fw_data, fw_size);
952
+
}
953
+
}
954
+
}
955
+
956
+
static void
957
+
gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
958
+
{
959
+
const __le32 *fw_data;
960
+
uint32_t fw_size;
961
+
const struct sdma_firmware_header_v3_0 *sdma_hdr;
962
+
963
+
sdma_hdr = (const struct sdma_firmware_header_v3_0 *)
964
+
adev->sdma.instance[0].fw->data;
965
+
fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
966
+
le32_to_cpu(sdma_hdr->ucode_offset_bytes));
967
+
fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes);
968
+
969
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
970
+
fw_data, fw_size);
971
+
}
972
+
973
+
static void
974
+
gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
975
+
{
976
+
const __le32 *fw_data;
977
+
unsigned fw_size;
978
+
const struct mes_firmware_header_v1_0 *mes_hdr;
979
+
int pipe, ucode_id, data_id;
980
+
981
+
for (pipe = 0; pipe < 2; pipe++) {
982
+
if (pipe == 0) {
983
+
ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0;
984
+
data_id = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK;
985
+
} else {
986
+
ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1;
987
+
data_id = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK;
988
+
}
989
+
990
+
mes_hdr = (const struct mes_firmware_header_v1_0 *)
991
+
adev->mes.fw[pipe]->data;
992
+
993
+
fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
994
+
le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
995
+
fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
996
+
997
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
998
+
999
+
fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1000
+
le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1001
+
fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1002
+
1003
+
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
1004
+
}
1005
+
}
1006
+
1007
+
static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1008
+
{
1009
+
uint32_t rlc_g_offset, rlc_g_size;
1010
+
uint64_t gpu_addr;
1011
+
uint32_t data;
1012
+
int i, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1013
+
1014
+
/* RLC autoload sequence 2: copy ucode */
1015
+
gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(adev);
1016
+
gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(adev);
1017
+
gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(adev);
1018
+
gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(adev);
1019
+
1020
+
rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset;
1021
+
rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size;
1022
+
gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
1023
+
1024
+
for (i = 0; i < num_xcc; i++) {
1025
+
WREG32_SOC15(GC, GET_INST(GC, i),
1026
+
regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1027
+
WREG32_SOC15(GC, GET_INST(GC, i),
1028
+
regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1029
+
1030
+
WREG32_SOC15(GC, GET_INST(GC, i),
1031
+
regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1032
+
1033
+
if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
1034
+
/* RLC autoload sequence 3: load IMU fw */
1035
+
if (adev->gfx.imu.funcs->load_microcode)
1036
+
adev->gfx.imu.funcs->load_microcode(adev);
1037
+
/* RLC autoload sequence 4 init IMU fw */
1038
+
if (adev->gfx.imu.funcs->setup_imu)
1039
+
adev->gfx.imu.funcs->setup_imu(adev);
1040
+
if (adev->gfx.imu.funcs->start_imu)
1041
+
adev->gfx.imu.funcs->start_imu(adev);
1042
+
1043
+
/* RLC autoload sequence 5 disable gpa mode */
1044
+
gfx_v12_1_xcc_disable_gpa_mode(adev, i);
1045
+
} else {
1046
+
/* unhalt rlc to start autoload without imu */
1047
+
data = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE);
1048
+
data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1);
1049
+
data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1050
+
WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE, data);
1051
+
WREG32_SOC15(GC, GET_INST(GC, i), regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
1052
+
}
1053
+
}
1054
+
1055
+
return 0;
1056
+
}
1057
+
1058
+
static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
1059
+
{
1060
+
int i, j, k, r, ring_id = 0;
1061
+
unsigned num_compute_rings;
1062
+
int xcc_id, num_xcc;
1063
+
struct amdgpu_device *adev = ip_block->adev;
1064
+
1065
+
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1066
+
case IP_VERSION(12, 1, 0):
1067
+
adev->gfx.mec.num_mec = 1;
1068
+
adev->gfx.mec.num_pipe_per_mec = 4;
1069
+
adev->gfx.mec.num_queue_per_pipe = 8;
1070
+
break;
1071
+
default:
1072
+
adev->gfx.mec.num_mec = 2;
1073
+
adev->gfx.mec.num_pipe_per_mec = 2;
1074
+
adev->gfx.mec.num_queue_per_pipe = 4;
1075
+
break;
1076
+
}
1077
+
1078
+
/* recalculate compute rings to use based on hardware configuration */
1079
+
num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
1080
+
adev->gfx.mec.num_queue_per_pipe) / 2;
1081
+
adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
1082
+
num_compute_rings);
1083
+
1084
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1085
+
1086
+
/* EOP Event */
1087
+
r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1088
+
GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1089
+
&adev->gfx.eop_irq);
1090
+
if (r)
1091
+
return r;
1092
+
1093
+
/* Privileged reg */
1094
+
r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1095
+
GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1096
+
&adev->gfx.priv_reg_irq);
1097
+
if (r)
1098
+
return r;
1099
+
1100
+
/* Privileged inst */
1101
+
r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1102
+
GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1103
+
&adev->gfx.priv_inst_irq);
1104
+
if (r)
1105
+
return r;
1106
+
1107
+
adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1108
+
1109
+
r = gfx_v12_1_rlc_init(adev);
1110
+
if (r) {
1111
+
dev_err(adev->dev, "Failed to init rlc BOs!\n");
1112
+
return r;
1113
+
}
1114
+
1115
+
r = gfx_v12_1_mec_init(adev);
1116
+
if (r) {
1117
+
dev_err(adev->dev, "Failed to init MEC BOs!\n");
1118
+
return r;
1119
+
}
1120
+
1121
+
/* set up the compute queues - allocate horizontally across pipes */
1122
+
for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1123
+
ring_id = 0;
1124
+
for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1125
+
for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1126
+
for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1127
+
if (!amdgpu_gfx_is_mec_queue_enabled(adev,
1128
+
xcc_id, i, k, j))
1129
+
continue;
1130
+
1131
+
r = gfx_v12_1_compute_ring_init(adev, ring_id,
1132
+
xcc_id, i, k, j);
1133
+
if (r)
1134
+
return r;
1135
+
1136
+
ring_id++;
1137
+
}
1138
+
}
1139
+
}
1140
+
1141
+
if (!adev->enable_mes_kiq) {
1142
+
r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, xcc_id);
1143
+
if (r) {
1144
+
dev_err(adev->dev, "Failed to init KIQ BOs!\n");
1145
+
return r;
1146
+
}
1147
+
1148
+
r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1149
+
if (r)
1150
+
return r;
1151
+
}
1152
+
1153
+
r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_1_compute_mqd), xcc_id);
1154
+
if (r)
1155
+
return r;
1156
+
}
1157
+
1158
+
/* allocate visible FB for rlc auto-loading fw */
1159
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1160
+
r = gfx_v12_1_rlc_autoload_buffer_init(adev);
1161
+
if (r)
1162
+
return r;
1163
+
}
1164
+
1165
+
r = gfx_v12_1_gpu_early_init(adev);
1166
+
if (r)
1167
+
return r;
1168
+
1169
+
return 0;
1170
+
}
1171
+
1172
+
static void gfx_v12_1_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1173
+
{
1174
+
amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1175
+
&adev->gfx.rlc.rlc_autoload_gpu_addr,
1176
+
(void **)&adev->gfx.rlc.rlc_autoload_ptr);
1177
+
}
1178
+
1179
+
static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
1180
+
{
1181
+
int i, num_xcc;
1182
+
struct amdgpu_device *adev = ip_block->adev;
1183
+
1184
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1185
+
for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
1186
+
amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1187
+
1188
+
for (i = 0; i < num_xcc; i++) {
1189
+
amdgpu_gfx_mqd_sw_fini(adev, i);
1190
+
1191
+
if (!adev->enable_mes_kiq) {
1192
+
amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring);
1193
+
amdgpu_gfx_kiq_fini(adev, i);
1194
+
}
1195
+
}
1196
+
1197
+
gfx_v12_1_rlc_fini(adev);
1198
+
gfx_v12_1_mec_fini(adev);
1199
+
1200
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1201
+
gfx_v12_1_rlc_autoload_buffer_fini(adev);
1202
+
1203
+
gfx_v12_1_free_microcode(adev);
1204
+
1205
+
return 0;
1206
+
}
1207
+
1208
+
static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1209
+
u32 sh_num, u32 instance, int xcc_id)
1210
+
{
1211
+
u32 data;
1212
+
1213
+
if (instance == 0xffffffff)
1214
+
data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1215
+
INSTANCE_BROADCAST_WRITES, 1);
1216
+
else
1217
+
data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1218
+
instance);
1219
+
1220
+
if (se_num == 0xffffffff)
1221
+
data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1222
+
1);
1223
+
else
1224
+
data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1225
+
1226
+
if (sh_num == 0xffffffff)
1227
+
data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1228
+
1);
1229
+
else
1230
+
data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1231
+
1232
+
WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data);
1233
+
}
1234
+
1235
+
static u32 gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device *adev,
1236
+
int xcc_id)
1237
+
{
1238
+
u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1239
+
1240
+
gc_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SA_UNIT_DISABLE);
1241
+
gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1242
+
CC_GC_SA_UNIT_DISABLE,
1243
+
SA_DISABLE);
1244
+
gc_user_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SA_UNIT_DISABLE);
1245
+
gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1246
+
GC_USER_SA_UNIT_DISABLE,
1247
+
SA_DISABLE);
1248
+
sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1249
+
adev->gfx.config.max_shader_engines);
1250
+
1251
+
return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1252
+
}
1253
+
1254
+
static u32 gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device *adev,
1255
+
int xcc_id)
1256
+
{
1257
+
u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1258
+
u32 rb_mask;
1259
+
1260
+
gc_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
1261
+
regCC_RB_BACKEND_DISABLE);
1262
+
gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1263
+
CC_RB_BACKEND_DISABLE,
1264
+
BACKEND_DISABLE);
1265
+
gc_user_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
1266
+
regGC_USER_RB_BACKEND_DISABLE);
1267
+
gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1268
+
GC_USER_RB_BACKEND_DISABLE,
1269
+
BACKEND_DISABLE);
1270
+
rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1271
+
adev->gfx.config.max_shader_engines);
1272
+
1273
+
return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1274
+
}
1275
+
1276
+
static void gfx_v12_1_setup_rb(struct amdgpu_device *adev)
1277
+
{
1278
+
u32 rb_bitmap_width_per_sa;
1279
+
u32 max_sa;
1280
+
u32 active_sa_bitmap;
1281
+
u32 global_active_rb_bitmap;
1282
+
u32 active_rb_bitmap = 0;
1283
+
u32 i;
1284
+
int xcc_id;
1285
+
1286
+
for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
1287
+
/* query sa bitmap from SA_UNIT_DISABLE registers */
1288
+
active_sa_bitmap = gfx_v12_1_get_sa_active_bitmap(adev, xcc_id);
1289
+
/* query rb bitmap from RB_BACKEND_DISABLE registers */
1290
+
global_active_rb_bitmap = gfx_v12_1_get_rb_active_bitmap(adev, xcc_id);
1291
+
1292
+
/* generate active rb bitmap according to active sa bitmap */
1293
+
max_sa = adev->gfx.config.max_shader_engines *
1294
+
adev->gfx.config.max_sh_per_se;
1295
+
rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1296
+
adev->gfx.config.max_sh_per_se;
1297
+
for (i = 0; i < max_sa; i++) {
1298
+
if (active_sa_bitmap & (1 << i))
1299
+
active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1300
+
}
1301
+
1302
+
active_rb_bitmap |= global_active_rb_bitmap;
1303
+
}
1304
+
1305
+
adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1306
+
adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1307
+
}
1308
+
1309
+
#define LDS_APP_BASE 0x1
1310
+
#define SCRATCH_APP_BASE 0x2
1311
+
1312
+
static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
1313
+
int xcc_id)
1314
+
{
1315
+
int i;
1316
+
uint32_t sh_mem_bases;
1317
+
uint32_t data;
1318
+
1319
+
/*
1320
+
* Configure apertures:
1321
+
* LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
1322
+
* Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
1323
+
* GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
1324
+
*/
1325
+
sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1326
+
SCRATCH_APP_BASE;
1327
+
1328
+
mutex_lock(&adev->srbm_mutex);
1329
+
for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1330
+
soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1331
+
/* CP and shaders */
1332
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1333
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases);
1334
+
1335
+
/* Enable trap for each kfd vmid. */
1336
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL);
1337
+
data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1338
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data);
1339
+
}
1340
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1341
+
mutex_unlock(&adev->srbm_mutex);
1342
+
}
1343
+
1344
+
static void gfx_v12_1_tcp_harvest(struct amdgpu_device *adev)
1345
+
{
1346
+
/* TODO: harvest feature to be added later. */
1347
+
}
1348
+
1349
+
static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
1350
+
{
1351
+
}
1352
+
1353
+
static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
1354
+
int xcc_id)
1355
+
{
1356
+
u32 tmp;
1357
+
int i;
1358
+
1359
+
/* XXX SH_MEM regs */
1360
+
/* where to put LDS, scratch, GPUVM in FSA64 space */
1361
+
mutex_lock(&adev->srbm_mutex);
1362
+
for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1363
+
soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1364
+
/* CP and shaders */
1365
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1366
+
regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1367
+
if (i != 0) {
1368
+
tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1369
+
(adev->gmc.private_aperture_start >> 48));
1370
+
tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1371
+
(adev->gmc.shared_aperture_start >> 48));
1372
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, tmp);
1373
+
}
1374
+
}
1375
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
1376
+
1377
+
mutex_unlock(&adev->srbm_mutex);
1378
+
1379
+
gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id);
1380
+
}
1381
+
1382
+
static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
1383
+
{
1384
+
int i, num_xcc;
1385
+
1386
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1387
+
1388
+
gfx_v12_1_setup_rb(adev);
1389
+
gfx_v12_1_get_cu_info(adev, &adev->gfx.cu_info);
1390
+
gfx_v12_1_get_tcc_info(adev);
1391
+
adev->gfx.config.pa_sc_tile_steering_override = 0;
1392
+
1393
+
for (i = 0; i < num_xcc; i++)
1394
+
gfx_v12_1_xcc_constants_init(adev, i);
1395
+
}
1396
+
1397
+
static void gfx_v12_1_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1398
+
bool enable, int xcc_id)
1399
+
{
1400
+
u32 tmp;
1401
+
1402
+
if (amdgpu_sriov_vf(adev))
1403
+
return;
1404
+
1405
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0);
1406
+
1407
+
tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1408
+
enable ? 1 : 0);
1409
+
tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1410
+
enable ? 1 : 0);
1411
+
tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1412
+
enable ? 1 : 0);
1413
+
tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1414
+
enable ? 1 : 0);
1415
+
1416
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp);
1417
+
}
1418
+
1419
+
static int gfx_v12_1_xcc_init_csb(struct amdgpu_device *adev,
1420
+
int xcc_id)
1421
+
{
1422
+
adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1423
+
1424
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_HI,
1425
+
adev->gfx.rlc.clear_state_gpu_addr >> 32);
1426
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_LO,
1427
+
adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1428
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1429
+
regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1430
+
1431
+
return 0;
1432
+
}
1433
+
1434
+
static void gfx_v12_1_xcc_rlc_stop(struct amdgpu_device *adev,
1435
+
int xcc_id)
1436
+
{
1437
+
u32 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL);
1438
+
1439
+
tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1440
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL, tmp);
1441
+
}
1442
+
1443
+
static void gfx_v12_1_rlc_stop(struct amdgpu_device *adev)
1444
+
{
1445
+
int i, num_xcc;
1446
+
1447
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1448
+
for (i = 0; i < num_xcc; i++)
1449
+
gfx_v12_1_xcc_rlc_stop(adev, i);
1450
+
}
1451
+
1452
+
static void gfx_v12_1_xcc_rlc_reset(struct amdgpu_device *adev,
1453
+
int xcc_id)
1454
+
{
1455
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1456
+
GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1457
+
udelay(50);
1458
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1459
+
GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1460
+
udelay(50);
1461
+
}
1462
+
1463
+
static void gfx_v12_1_rlc_reset(struct amdgpu_device *adev)
1464
+
{
1465
+
int i, num_xcc;
1466
+
1467
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1468
+
for (i = 0; i < num_xcc; i++)
1469
+
gfx_v12_1_xcc_rlc_reset(adev, i);
1470
+
}
1471
+
1472
+
static void gfx_v12_1_xcc_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1473
+
bool enable, int xcc_id)
1474
+
{
1475
+
uint32_t rlc_pg_cntl;
1476
+
1477
+
rlc_pg_cntl = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL);
1478
+
1479
+
if (!enable) {
1480
+
/* RLC_PG_CNTL[23] = 0 (default)
1481
+
* RLC will wait for handshake acks with SMU
1482
+
* GFXOFF will be enabled
1483
+
* RLC_PG_CNTL[23] = 1
1484
+
* RLC will not issue any message to SMU
1485
+
* hence no handshake between SMU & RLC
1486
+
* GFXOFF will be disabled
1487
+
*/
1488
+
rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1489
+
} else
1490
+
rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1491
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, rlc_pg_cntl);
1492
+
}
1493
+
1494
+
static void gfx_v12_1_xcc_rlc_start(struct amdgpu_device *adev,
1495
+
int xcc_id)
1496
+
{
1497
+
/* TODO: enable rlc & smu handshake until smu
1498
+
* and gfxoff feature works as expected */
1499
+
if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1500
+
gfx_v12_1_xcc_rlc_smu_handshake_cntl(adev, false, xcc_id);
1501
+
1502
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL, RLC_ENABLE_F32, 1);
1503
+
udelay(50);
1504
+
}
1505
+
1506
+
static void gfx_v12_1_rlc_start(struct amdgpu_device *adev)
1507
+
{
1508
+
int i, num_xcc;
1509
+
1510
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1511
+
for (i = 0; i < num_xcc; i++) {
1512
+
gfx_v12_1_xcc_rlc_start(adev, i);
1513
+
}
1514
+
}
1515
+
1516
+
static void gfx_v12_1_xcc_rlc_enable_srm(struct amdgpu_device *adev,
1517
+
int xcc_id)
1518
+
{
1519
+
uint32_t tmp;
1520
+
1521
+
/* enable Save Restore Machine */
1522
+
tmp = RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL));
1523
+
tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1524
+
tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1525
+
WREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL), tmp);
1526
+
}
1527
+
1528
+
static void gfx_v12_1_xcc_load_rlcg_microcode(struct amdgpu_device *adev,
1529
+
int xcc_id)
1530
+
{
1531
+
const struct rlc_firmware_header_v2_0 *hdr;
1532
+
const __le32 *fw_data;
1533
+
unsigned i, fw_size;
1534
+
1535
+
hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1536
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1537
+
le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1538
+
fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1539
+
1540
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR,
1541
+
RLCG_UCODE_LOADING_START_ADDRESS);
1542
+
1543
+
for (i = 0; i < fw_size; i++)
1544
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1545
+
regRLC_GPM_UCODE_DATA,
1546
+
le32_to_cpup(fw_data++));
1547
+
1548
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1549
+
regRLC_GPM_UCODE_ADDR,
1550
+
adev->gfx.rlc_fw_version);
1551
+
}
1552
+
1553
+
static void gfx_v12_1_xcc_load_rlc_iram_dram_microcode(struct amdgpu_device *adev,
1554
+
int xcc_id)
1555
+
{
1556
+
const struct rlc_firmware_header_v2_2 *hdr;
1557
+
const __le32 *fw_data;
1558
+
unsigned i, fw_size;
1559
+
u32 tmp;
1560
+
1561
+
hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1562
+
1563
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1564
+
le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1565
+
fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1566
+
1567
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_IRAM_ADDR, 0);
1568
+
1569
+
for (i = 0; i < fw_size; i++) {
1570
+
if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1571
+
msleep(1);
1572
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1573
+
regRLC_LX6_IRAM_DATA,
1574
+
le32_to_cpup(fw_data++));
1575
+
}
1576
+
1577
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1578
+
regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1579
+
1580
+
fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1581
+
le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
1582
+
fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
1583
+
1584
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1585
+
regRLC_LX6_DRAM_ADDR, 0);
1586
+
for (i = 0; i < fw_size; i++) {
1587
+
if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1588
+
msleep(1);
1589
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1590
+
regRLC_LX6_DRAM_DATA,
1591
+
le32_to_cpup(fw_data++));
1592
+
}
1593
+
1594
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1595
+
regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1596
+
1597
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL);
1598
+
tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
1599
+
tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
1600
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL, tmp);
1601
+
}
1602
+
1603
+
static int gfx_v12_1_xcc_rlc_load_microcode(struct amdgpu_device *adev,
1604
+
int xcc_id)
1605
+
{
1606
+
const struct rlc_firmware_header_v2_0 *hdr;
1607
+
uint16_t version_major;
1608
+
uint16_t version_minor;
1609
+
1610
+
if (!adev->gfx.rlc_fw)
1611
+
return -EINVAL;
1612
+
1613
+
hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1614
+
amdgpu_ucode_print_rlc_hdr(&hdr->header);
1615
+
1616
+
version_major = le16_to_cpu(hdr->header.header_version_major);
1617
+
version_minor = le16_to_cpu(hdr->header.header_version_minor);
1618
+
1619
+
if (version_major == 2) {
1620
+
gfx_v12_1_xcc_load_rlcg_microcode(adev, xcc_id);
1621
+
if (amdgpu_dpm == 1) {
1622
+
if (version_minor >= 2)
1623
+
gfx_v12_1_xcc_load_rlc_iram_dram_microcode(adev, xcc_id);
1624
+
}
1625
+
1626
+
return 0;
1627
+
}
1628
+
1629
+
return -EINVAL;
1630
+
}
1631
+
1632
+
static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev,
1633
+
int xcc_id)
1634
+
{
1635
+
int r;
1636
+
1637
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1638
+
gfx_v12_1_xcc_init_csb(adev, xcc_id);
1639
+
1640
+
if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
1641
+
gfx_v12_1_xcc_rlc_enable_srm(adev, xcc_id);
1642
+
} else {
1643
+
if (amdgpu_sriov_vf(adev)) {
1644
+
gfx_v12_1_xcc_init_csb(adev, xcc_id);
1645
+
return 0;
1646
+
}
1647
+
1648
+
gfx_v12_1_xcc_rlc_stop(adev, xcc_id);
1649
+
1650
+
/* disable CG */
1651
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0);
1652
+
1653
+
/* disable PG */
1654
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, 0);
1655
+
1656
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1657
+
/* legacy rlc firmware loading */
1658
+
r = gfx_v12_1_xcc_rlc_load_microcode(adev, xcc_id);
1659
+
if (r)
1660
+
return r;
1661
+
}
1662
+
1663
+
gfx_v12_1_xcc_init_csb(adev, xcc_id);
1664
+
1665
+
gfx_v12_1_xcc_rlc_start(adev, xcc_id);
1666
+
}
1667
+
1668
+
return 0;
1669
+
}
1670
+
1671
+
static int gfx_v12_1_rlc_resume(struct amdgpu_device *adev)
1672
+
{
1673
+
int r, i, num_xcc;
1674
+
1675
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1676
+
for (i = 0; i < num_xcc; i++) {
1677
+
r = gfx_v12_1_xcc_rlc_resume(adev, i);
1678
+
if (r)
1679
+
return r;
1680
+
}
1681
+
1682
+
return 0;
1683
+
}
1684
+
1685
+
static void gfx_v12_1_xcc_config_gfx_rs64(struct amdgpu_device *adev,
1686
+
int xcc_id)
1687
+
{
1688
+
const struct gfx_firmware_header_v2_0 *mec_hdr;
1689
+
uint32_t pipe_id, tmp;
1690
+
1691
+
mec_hdr = (const struct gfx_firmware_header_v2_0 *)
1692
+
adev->gfx.mec_fw->data;
1693
+
1694
+
/* config mec program start addr */
1695
+
for (pipe_id = 0; pipe_id < 4; pipe_id++) {
1696
+
soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
1697
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
1698
+
mec_hdr->ucode_start_addr_lo >> 2 |
1699
+
mec_hdr->ucode_start_addr_hi << 30);
1700
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
1701
+
mec_hdr->ucode_start_addr_hi >> 2);
1702
+
}
1703
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
1704
+
1705
+
/* reset mec pipe */
1706
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
1707
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
1708
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
1709
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
1710
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
1711
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
1712
+
1713
+
/* clear mec pipe reset */
1714
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
1715
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
1716
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
1717
+
tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
1718
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
1719
+
}
1720
+
1721
+
static void gfx_v12_1_config_gfx_rs64(struct amdgpu_device *adev)
1722
+
{
1723
+
int i, num_xcc;
1724
+
1725
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1726
+
1727
+
for (i = 0; i < num_xcc; i++)
1728
+
gfx_v12_1_xcc_config_gfx_rs64(adev, i);
1729
+
}
1730
+
1731
+
static void gfx_v12_1_xcc_set_mec_ucode_start_addr(struct amdgpu_device *adev,
1732
+
int xcc_id)
1733
+
{
1734
+
const struct gfx_firmware_header_v2_0 *cp_hdr;
1735
+
unsigned pipe_id;
1736
+
1737
+
cp_hdr = (const struct gfx_firmware_header_v2_0 *)
1738
+
adev->gfx.mec_fw->data;
1739
+
mutex_lock(&adev->srbm_mutex);
1740
+
for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
1741
+
soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
1742
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
1743
+
cp_hdr->ucode_start_addr_lo >> 2 |
1744
+
cp_hdr->ucode_start_addr_hi << 30);
1745
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
1746
+
cp_hdr->ucode_start_addr_hi >> 2);
1747
+
}
1748
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
1749
+
mutex_unlock(&adev->srbm_mutex);
1750
+
}
1751
+
1752
+
static int gfx_v12_1_xcc_wait_for_rlc_autoload_complete(struct amdgpu_device *adev,
1753
+
int xcc_id)
1754
+
{
1755
+
uint32_t cp_status;
1756
+
uint32_t bootload_status;
1757
+
int i;
1758
+
1759
+
for (i = 0; i < adev->usec_timeout; i++) {
1760
+
cp_status = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_STAT);
1761
+
bootload_status = RREG32_SOC15(GC, GET_INST(GC, 0),
1762
+
regRLC_RLCS_BOOTLOAD_STATUS);
1763
+
1764
+
if ((cp_status == 0) &&
1765
+
(REG_GET_FIELD(bootload_status,
1766
+
RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
1767
+
break;
1768
+
}
1769
+
udelay(1);
1770
+
if (amdgpu_emu_mode)
1771
+
msleep(10);
1772
+
}
1773
+
1774
+
if (i >= adev->usec_timeout) {
1775
+
dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
1776
+
return -ETIMEDOUT;
1777
+
}
1778
+
1779
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1780
+
gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
1781
+
}
1782
+
1783
+
return 0;
1784
+
}
1785
+
1786
+
static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
1787
+
{
1788
+
int xcc_id, r;
1789
+
1790
+
for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
1791
+
r = gfx_v12_1_xcc_wait_for_rlc_autoload_complete(adev, xcc_id);
1792
+
if (r)
1793
+
return r;
1794
+
}
1795
+
1796
+
return 0;
1797
+
}
1798
+
1799
+
static void gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device *adev,
1800
+
bool enable, int xcc_id)
1801
+
{
1802
+
u32 data;
1803
+
1804
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
1805
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
1806
+
enable ? 0 : 1);
1807
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
1808
+
enable ? 0 : 1);
1809
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
1810
+
enable ? 0 : 1);
1811
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
1812
+
enable ? 0 : 1);
1813
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
1814
+
enable ? 0 : 1);
1815
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
1816
+
enable ? 1 : 0);
1817
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
1818
+
enable ? 1 : 0);
1819
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
1820
+
enable ? 1 : 0);
1821
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
1822
+
enable ? 1 : 0);
1823
+
data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
1824
+
enable ? 0 : 1);
1825
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, data);
1826
+
1827
+
adev->gfx.kiq[xcc_id].ring.sched.ready = enable;
1828
+
1829
+
udelay(50);
1830
+
}
1831
+
1832
+
static int gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device *adev,
1833
+
int xcc_id)
1834
+
{
1835
+
const struct gfx_firmware_header_v2_0 *mec_hdr;
1836
+
const __le32 *fw_ucode, *fw_data;
1837
+
u32 tmp, fw_ucode_size, fw_data_size;
1838
+
u32 i, usec_timeout = 50000; /* Wait for 50 ms */
1839
+
u32 *fw_ucode_ptr, *fw_data_ptr;
1840
+
int r;
1841
+
1842
+
if (!adev->gfx.mec_fw)
1843
+
return -EINVAL;
1844
+
1845
+
gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
1846
+
1847
+
mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
1848
+
amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
1849
+
1850
+
fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
1851
+
le32_to_cpu(mec_hdr->ucode_offset_bytes));
1852
+
fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
1853
+
1854
+
fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1855
+
le32_to_cpu(mec_hdr->data_offset_bytes));
1856
+
fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
1857
+
1858
+
r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
1859
+
64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
1860
+
&adev->gfx.mec.mec_fw_obj,
1861
+
&adev->gfx.mec.mec_fw_gpu_addr,
1862
+
(void **)&fw_ucode_ptr);
1863
+
if (r) {
1864
+
dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
1865
+
gfx_v12_1_mec_fini(adev);
1866
+
return r;
1867
+
}
1868
+
1869
+
r = amdgpu_bo_create_reserved(adev,
1870
+
ALIGN(fw_data_size, 64 * 1024) *
1871
+
adev->gfx.mec.num_pipe_per_mec,
1872
+
64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
1873
+
&adev->gfx.mec.mec_fw_data_obj,
1874
+
&adev->gfx.mec.mec_fw_data_gpu_addr,
1875
+
(void **)&fw_data_ptr);
1876
+
if (r) {
1877
+
dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
1878
+
gfx_v12_1_mec_fini(adev);
1879
+
return r;
1880
+
}
1881
+
1882
+
memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
1883
+
for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
1884
+
memcpy(fw_data_ptr + i * ALIGN(fw_data_size, 64 * 1024) / 4, fw_data, fw_data_size);
1885
+
}
1886
+
1887
+
amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1888
+
amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
1889
+
amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1890
+
amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
1891
+
1892
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL);
1893
+
tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
1894
+
tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
1895
+
tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
1896
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp);
1897
+
1898
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL);
1899
+
tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
1900
+
tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
1901
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL, tmp);
1902
+
1903
+
mutex_lock(&adev->srbm_mutex);
1904
+
for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
1905
+
soc_v1_0_grbm_select(adev, 1, i, 0, 0, GET_INST(GC, xcc_id));
1906
+
1907
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_LO,
1908
+
lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
1909
+
i * ALIGN(fw_data_size, 64 * 1024)));
1910
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_HI,
1911
+
upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
1912
+
i * ALIGN(fw_data_size, 64 * 1024)));
1913
+
1914
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO,
1915
+
lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1916
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI,
1917
+
upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1918
+
}
1919
+
mutex_unlock(&adev->srbm_mutex);
1920
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
1921
+
1922
+
/* Trigger an invalidation of the L1 instruction caches */
1923
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
1924
+
tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
1925
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL, tmp);
1926
+
1927
+
/* Wait for invalidation complete */
1928
+
for (i = 0; i < usec_timeout; i++) {
1929
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
1930
+
if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
1931
+
INVALIDATE_DCACHE_COMPLETE))
1932
+
break;
1933
+
udelay(1);
1934
+
}
1935
+
1936
+
if (i >= usec_timeout) {
1937
+
dev_err(adev->dev, "failed to invalidate instruction cache\n");
1938
+
return -EINVAL;
1939
+
}
1940
+
1941
+
/* Trigger an invalidation of the L1 instruction caches */
1942
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
1943
+
tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1944
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL, tmp);
1945
+
1946
+
/* Wait for invalidation complete */
1947
+
for (i = 0; i < usec_timeout; i++) {
1948
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
1949
+
if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
1950
+
INVALIDATE_CACHE_COMPLETE))
1951
+
break;
1952
+
udelay(1);
1953
+
}
1954
+
1955
+
if (i >= usec_timeout) {
1956
+
dev_err(adev->dev, "failed to invalidate instruction cache\n");
1957
+
return -EINVAL;
1958
+
}
1959
+
1960
+
gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
1961
+
1962
+
return 0;
1963
+
}
1964
+
1965
+
static void gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring *ring,
1966
+
int xcc_id)
1967
+
{
1968
+
uint32_t tmp;
1969
+
struct amdgpu_device *adev = ring->adev;
1970
+
1971
+
/* tell RLC which is KIQ queue */
1972
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
1973
+
tmp &= 0xffffff00;
1974
+
tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1975
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
1976
+
tmp |= 0x80;
1977
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
1978
+
}
1979
+
1980
+
static void gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device *adev,
1981
+
int xcc_id)
1982
+
{
1983
+
/* set graphics engine doorbell range */
1984
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_LOWER,
1985
+
(adev->doorbell_index.gfx_ring0 * 2) << 2);
1986
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_UPPER,
1987
+
(adev->doorbell_index.gfx_userqueue_end * 2) << 2);
1988
+
1989
+
/* set compute engine doorbell range */
1990
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_LOWER,
1991
+
(adev->doorbell_index.kiq * 2) << 2);
1992
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_UPPER,
1993
+
(adev->doorbell_index.userqueue_end * 2) << 2);
1994
+
}
1995
+
1996
+
static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
1997
+
struct amdgpu_mqd_prop *prop)
1998
+
{
1999
+
struct v12_1_compute_mqd *mqd = m;
2000
+
uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
2001
+
uint32_t tmp;
2002
+
2003
+
mqd->header = 0xC0310800;
2004
+
mqd->compute_pipelinestat_enable = 0x00000001;
2005
+
mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2006
+
mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2007
+
mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2008
+
mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2009
+
mqd->compute_misc_reserved = 0x00000007;
2010
+
2011
+
eop_base_addr = prop->eop_gpu_addr >> 8;
2012
+
mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
2013
+
mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
2014
+
2015
+
/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2016
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_EOP_CONTROL);
2017
+
tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
2018
+
(order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1));
2019
+
2020
+
mqd->cp_hqd_eop_control = tmp;
2021
+
2022
+
/* enable doorbell? */
2023
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_DOORBELL_CONTROL);
2024
+
2025
+
if (prop->use_doorbell) {
2026
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2027
+
DOORBELL_OFFSET, prop->doorbell_index);
2028
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2029
+
DOORBELL_EN, 1);
2030
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2031
+
DOORBELL_SOURCE, 0);
2032
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2033
+
DOORBELL_HIT, 0);
2034
+
} else {
2035
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2036
+
DOORBELL_EN, 0);
2037
+
}
2038
+
2039
+
mqd->cp_hqd_pq_doorbell_control = tmp;
2040
+
2041
+
/* disable the queue if it's active */
2042
+
mqd->cp_hqd_dequeue_request = 0;
2043
+
mqd->cp_hqd_pq_rptr = 0;
2044
+
mqd->cp_hqd_pq_wptr_lo = 0;
2045
+
mqd->cp_hqd_pq_wptr_hi = 0;
2046
+
2047
+
/* set the pointer to the MQD */
2048
+
mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
2049
+
mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2050
+
2051
+
/* set MQD vmid to 0 */
2052
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_MQD_CONTROL);
2053
+
tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
2054
+
mqd->cp_mqd_control = tmp;
2055
+
2056
+
/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2057
+
hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2058
+
mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2059
+
mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2060
+
2061
+
/* set up the HQD, this is similar to CP_RB0_CNTL */
2062
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_CONTROL);
2063
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
2064
+
(order_base_2(prop->queue_size / 4) - 1));
2065
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
2066
+
(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
2067
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
2068
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
2069
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
2070
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
2071
+
mqd->cp_hqd_pq_control = tmp;
2072
+
2073
+
/* set the wb address whether it's enabled or not */
2074
+
wb_gpu_addr = prop->rptr_gpu_addr;
2075
+
mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2076
+
mqd->cp_hqd_pq_rptr_report_addr_hi =
2077
+
upper_32_bits(wb_gpu_addr) & 0xffff;
2078
+
2079
+
/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2080
+
wb_gpu_addr = prop->wptr_gpu_addr;
2081
+
mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2082
+
mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2083
+
2084
+
tmp = 0;
2085
+
/* enable the doorbell if requested */
2086
+
if (prop->use_doorbell) {
2087
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_DOORBELL_CONTROL);
2088
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2089
+
DOORBELL_OFFSET, prop->doorbell_index);
2090
+
2091
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2092
+
DOORBELL_EN, 1);
2093
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2094
+
DOORBELL_SOURCE, 0);
2095
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2096
+
DOORBELL_HIT, 0);
2097
+
}
2098
+
2099
+
mqd->cp_hqd_pq_doorbell_control = tmp;
2100
+
2101
+
/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2102
+
mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_RPTR);
2103
+
2104
+
/* set the vmid for the queue */
2105
+
mqd->cp_hqd_vmid = 0;
2106
+
2107
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PERSISTENT_STATE);
2108
+
tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x63);
2109
+
mqd->cp_hqd_persistent_state = tmp;
2110
+
2111
+
/* set MIN_IB_AVAIL_SIZE */
2112
+
tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_IB_CONTROL);
2113
+
tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 1);
2114
+
mqd->cp_hqd_ib_control = tmp;
2115
+
2116
+
/* set static priority for a compute queue/ring */
2117
+
mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
2118
+
mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
2119
+
2120
+
mqd->cp_hqd_active = prop->hqd_active;
2121
+
2122
+
return 0;
2123
+
}
2124
+
2125
+
static int gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring *ring,
2126
+
int xcc_id)
2127
+
{
2128
+
struct amdgpu_device *adev = ring->adev;
2129
+
struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2130
+
int j;
2131
+
2132
+
/* inactivate the queue */
2133
+
if (amdgpu_sriov_vf(adev))
2134
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
2135
+
2136
+
/* disable wptr polling */
2137
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
2138
+
2139
+
/* write the EOP addr */
2140
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
2141
+
mqd->cp_hqd_eop_base_addr_lo);
2142
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
2143
+
mqd->cp_hqd_eop_base_addr_hi);
2144
+
2145
+
/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2146
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
2147
+
mqd->cp_hqd_eop_control);
2148
+
2149
+
/* enable doorbell? */
2150
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
2151
+
mqd->cp_hqd_pq_doorbell_control);
2152
+
2153
+
/* disable the queue if it's active */
2154
+
if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
2155
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
2156
+
for (j = 0; j < adev->usec_timeout; j++) {
2157
+
if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
2158
+
break;
2159
+
udelay(1);
2160
+
}
2161
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
2162
+
mqd->cp_hqd_dequeue_request);
2163
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
2164
+
mqd->cp_hqd_pq_rptr);
2165
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
2166
+
mqd->cp_hqd_pq_wptr_lo);
2167
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
2168
+
mqd->cp_hqd_pq_wptr_hi);
2169
+
}
2170
+
2171
+
/* set the pointer to the MQD */
2172
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
2173
+
mqd->cp_mqd_base_addr_lo);
2174
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
2175
+
mqd->cp_mqd_base_addr_hi);
2176
+
2177
+
/* set MQD vmid to 0 */
2178
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
2179
+
mqd->cp_mqd_control);
2180
+
2181
+
/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2182
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
2183
+
mqd->cp_hqd_pq_base_lo);
2184
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
2185
+
mqd->cp_hqd_pq_base_hi);
2186
+
2187
+
/* set up the HQD, this is similar to CP_RB0_CNTL */
2188
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
2189
+
mqd->cp_hqd_pq_control);
2190
+
2191
+
/* set the wb address whether it's enabled or not */
2192
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
2193
+
mqd->cp_hqd_pq_rptr_report_addr_lo);
2194
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2195
+
mqd->cp_hqd_pq_rptr_report_addr_hi);
2196
+
2197
+
/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2198
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
2199
+
mqd->cp_hqd_pq_wptr_poll_addr_lo);
2200
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2201
+
mqd->cp_hqd_pq_wptr_poll_addr_hi);
2202
+
2203
+
/* enable the doorbell if requested */
2204
+
if (ring->use_doorbell) {
2205
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_LOWER,
2206
+
(adev->doorbell_index.kiq * 2) << 2);
2207
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_UPPER,
2208
+
(adev->doorbell_index.userqueue_end * 2) << 2);
2209
+
}
2210
+
2211
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
2212
+
mqd->cp_hqd_pq_doorbell_control);
2213
+
2214
+
/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2215
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
2216
+
mqd->cp_hqd_pq_wptr_lo);
2217
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
2218
+
mqd->cp_hqd_pq_wptr_hi);
2219
+
2220
+
/* set the vmid for the queue */
2221
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
2222
+
2223
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
2224
+
mqd->cp_hqd_persistent_state);
2225
+
2226
+
/* activate the queue */
2227
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
2228
+
mqd->cp_hqd_active);
2229
+
2230
+
if (ring->use_doorbell)
2231
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2232
+
2233
+
return 0;
2234
+
}
2235
+
2236
+
static int gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring *ring,
2237
+
int xcc_id)
2238
+
{
2239
+
struct amdgpu_device *adev = ring->adev;
2240
+
struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2241
+
2242
+
gfx_v12_1_xcc_kiq_setting(ring, xcc_id);
2243
+
2244
+
if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
2245
+
/* reset MQD to a clean status */
2246
+
if (adev->gfx.kiq[xcc_id].mqd_backup)
2247
+
memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(*mqd));
2248
+
2249
+
/* reset ring buffer */
2250
+
ring->wptr = 0;
2251
+
amdgpu_ring_clear_ring(ring);
2252
+
2253
+
mutex_lock(&adev->srbm_mutex);
2254
+
soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2255
+
gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
2256
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2257
+
mutex_unlock(&adev->srbm_mutex);
2258
+
} else {
2259
+
memset((void *)mqd, 0, sizeof(*mqd));
2260
+
if (amdgpu_sriov_vf(adev) && adev->in_suspend)
2261
+
amdgpu_ring_clear_ring(ring);
2262
+
mutex_lock(&adev->srbm_mutex);
2263
+
soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2264
+
amdgpu_ring_init_mqd(ring);
2265
+
gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
2266
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2267
+
mutex_unlock(&adev->srbm_mutex);
2268
+
2269
+
if (adev->gfx.kiq[xcc_id].mqd_backup)
2270
+
memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(*mqd));
2271
+
}
2272
+
2273
+
return 0;
2274
+
}
2275
+
2276
+
static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring,
2277
+
int xcc_id)
2278
+
{
2279
+
struct amdgpu_device *adev = ring->adev;
2280
+
struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2281
+
int mqd_idx = ring - &adev->gfx.compute_ring[0];
2282
+
2283
+
if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2284
+
memset((void *)mqd, 0, sizeof(*mqd));
2285
+
mutex_lock(&adev->srbm_mutex);
2286
+
soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2287
+
amdgpu_ring_init_mqd(ring);
2288
+
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2289
+
mutex_unlock(&adev->srbm_mutex);
2290
+
2291
+
if (adev->gfx.mec.mqd_backup[mqd_idx])
2292
+
memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
2293
+
} else {
2294
+
/* restore MQD to a clean status */
2295
+
if (adev->gfx.mec.mqd_backup[mqd_idx])
2296
+
memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
2297
+
/* reset ring buffer */
2298
+
ring->wptr = 0;
2299
+
atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
2300
+
amdgpu_ring_clear_ring(ring);
2301
+
}
2302
+
2303
+
return 0;
2304
+
}
2305
+
2306
+
static int gfx_v12_1_xcc_kiq_resume(struct amdgpu_device *adev,
2307
+
int xcc_id)
2308
+
{
2309
+
struct amdgpu_ring *ring;
2310
+
int r;
2311
+
2312
+
ring = &adev->gfx.kiq[xcc_id].ring;
2313
+
2314
+
r = amdgpu_bo_reserve(ring->mqd_obj, false);
2315
+
if (unlikely(r != 0))
2316
+
return r;
2317
+
2318
+
r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2319
+
if (unlikely(r != 0)) {
2320
+
amdgpu_bo_unreserve(ring->mqd_obj);
2321
+
return r;
2322
+
}
2323
+
2324
+
gfx_v12_1_xcc_kiq_init_queue(ring, xcc_id);
2325
+
amdgpu_bo_kunmap(ring->mqd_obj);
2326
+
ring->mqd_ptr = NULL;
2327
+
amdgpu_bo_unreserve(ring->mqd_obj);
2328
+
ring->sched.ready = true;
2329
+
return 0;
2330
+
}
2331
+
2332
+
static int gfx_v12_1_xcc_kcq_resume(struct amdgpu_device *adev,
2333
+
int xcc_id)
2334
+
{
2335
+
struct amdgpu_ring *ring = NULL;
2336
+
int r = 0, i;
2337
+
2338
+
if (!amdgpu_async_gfx_ring)
2339
+
gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
2340
+
2341
+
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2342
+
ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
2343
+
2344
+
r = amdgpu_bo_reserve(ring->mqd_obj, false);
2345
+
if (unlikely(r != 0))
2346
+
goto done;
2347
+
r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2348
+
if (!r) {
2349
+
r = gfx_v12_1_xcc_kcq_init_queue(ring, xcc_id);
2350
+
amdgpu_bo_kunmap(ring->mqd_obj);
2351
+
ring->mqd_ptr = NULL;
2352
+
}
2353
+
amdgpu_bo_unreserve(ring->mqd_obj);
2354
+
if (r)
2355
+
goto done;
2356
+
}
2357
+
2358
+
r = amdgpu_gfx_enable_kcq(adev, xcc_id);
2359
+
done:
2360
+
return r;
2361
+
}
2362
+
2363
+
static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev,
2364
+
int xcc_id)
2365
+
{
2366
+
int r, i;
2367
+
struct amdgpu_ring *ring;
2368
+
2369
+
if (!(adev->flags & AMD_IS_APU))
2370
+
gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
2371
+
2372
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2373
+
/* legacy firmware loading */
2374
+
r = gfx_v12_1_xcc_cp_compute_load_microcode_rs64(adev, xcc_id);
2375
+
if (r)
2376
+
return r;
2377
+
}
2378
+
2379
+
gfx_v12_1_xcc_cp_set_doorbell_range(adev, xcc_id);
2380
+
2381
+
if (amdgpu_async_gfx_ring) {
2382
+
gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
2383
+
}
2384
+
2385
+
if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
2386
+
r = amdgpu_mes_kiq_hw_init(adev, xcc_id);
2387
+
else
2388
+
r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id);
2389
+
if (r)
2390
+
return r;
2391
+
2392
+
r = gfx_v12_1_xcc_kcq_resume(adev, xcc_id);
2393
+
if (r)
2394
+
return r;
2395
+
2396
+
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2397
+
ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
2398
+
r = amdgpu_ring_test_helper(ring);
2399
+
if (r)
2400
+
return r;
2401
+
}
2402
+
2403
+
return 0;
2404
+
}
2405
+
2406
+
static int gfx_v12_1_cp_resume(struct amdgpu_device *adev)
2407
+
{
2408
+
int i, r, num_xcc;
2409
+
2410
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2411
+
2412
+
for (i = 0; i < num_xcc; i++) {
2413
+
r = gfx_v12_1_xcc_cp_resume(adev, i);
2414
+
if (r)
2415
+
return r;
2416
+
}
2417
+
2418
+
return 0;
2419
+
}
2420
+
2421
+
static int gfx_v12_1_gfxhub_enable(struct amdgpu_device *adev)
2422
+
{
2423
+
int r;
2424
+
bool value;
2425
+
2426
+
r = adev->gfxhub.funcs->gart_enable(adev);
2427
+
if (r)
2428
+
return r;
2429
+
2430
+
adev->hdp.funcs->flush_hdp(adev, NULL);
2431
+
2432
+
value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
2433
+
false : true;
2434
+
2435
+
adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2436
+
/* TODO investigate why this and the hdp flush above is needed,
2437
+
* are we missing a flush somewhere else? */
2438
+
adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
2439
+
2440
+
return 0;
2441
+
}
2442
+
2443
+
static int get_gb_addr_config(struct amdgpu_device *adev)
2444
+
{
2445
+
u32 gb_addr_config;
2446
+
2447
+
gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG_READ);
2448
+
if (gb_addr_config == 0)
2449
+
return -EINVAL;
2450
+
2451
+
adev->gfx.config.gb_addr_config_fields.num_pkrs =
2452
+
1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG_READ, NUM_PKRS);
2453
+
2454
+
adev->gfx.config.gb_addr_config = gb_addr_config;
2455
+
2456
+
adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2457
+
REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2458
+
GB_ADDR_CONFIG_READ, NUM_PIPES);
2459
+
2460
+
adev->gfx.config.max_tile_pipes =
2461
+
adev->gfx.config.gb_addr_config_fields.num_pipes;
2462
+
2463
+
adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2464
+
REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2465
+
GB_ADDR_CONFIG_READ, MAX_COMPRESSED_FRAGS);
2466
+
adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2467
+
REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2468
+
GB_ADDR_CONFIG_READ, NUM_RB_PER_SE);
2469
+
adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2470
+
REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2471
+
GB_ADDR_CONFIG_READ, NUM_SHADER_ENGINES);
2472
+
adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2473
+
REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2474
+
GB_ADDR_CONFIG_READ, PIPE_INTERLEAVE_SIZE));
2475
+
2476
+
return 0;
2477
+
}
2478
+
2479
+
static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev,
2480
+
int xcc_id)
2481
+
{
2482
+
uint32_t data;
2483
+
2484
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG);
2485
+
data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
2486
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data);
2487
+
2488
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG);
2489
+
data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
2490
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG, data);
2491
+
}
2492
+
2493
+
static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev)
2494
+
{
2495
+
int i, num_xcc;
2496
+
uint32_t val;
2497
+
2498
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2499
+
for (i = 0; i < num_xcc; i++) {
2500
+
/* Setup the TCP Thrashing control register */
2501
+
val = RREG32_SOC15(GC, GET_INST(GC, i), regTCP_UTCL0_THRASHING_CTRL);
2502
+
2503
+
val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL, THRASHING_EN, 0x2);
2504
+
val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL,
2505
+
RETRY_FRAGMENT_THRESHOLD_DOWN_EN, 0x0);
2506
+
val = REG_SET_FIELD(val, TCP_UTCL0_THRASHING_CTRL,
2507
+
RETRY_FRAGMENT_THRESHOLD_UP_EN, 0x0);
2508
+
2509
+
WREG32_SOC15(GC, GET_INST(GC, i), regTCP_UTCL0_THRASHING_CTRL, val);
2510
+
}
2511
+
}
2512
+
2513
+
static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
2514
+
{
2515
+
int r, i, num_xcc;
2516
+
struct amdgpu_device *adev = ip_block->adev;
2517
+
2518
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2519
+
if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
2520
+
/* RLC autoload sequence 1: Program rlc ram */
2521
+
if (adev->gfx.imu.funcs->program_rlc_ram)
2522
+
adev->gfx.imu.funcs->program_rlc_ram(adev);
2523
+
}
2524
+
/* rlc autoload firmware */
2525
+
r = gfx_v12_1_rlc_backdoor_autoload_enable(adev);
2526
+
if (r)
2527
+
return r;
2528
+
} else {
2529
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2530
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2531
+
2532
+
for (i = 0; i < num_xcc; i++) {
2533
+
if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
2534
+
if (adev->gfx.imu.funcs->load_microcode)
2535
+
adev->gfx.imu.funcs->load_microcode(adev);
2536
+
if (adev->gfx.imu.funcs->setup_imu)
2537
+
adev->gfx.imu.funcs->setup_imu(adev);
2538
+
if (adev->gfx.imu.funcs->start_imu)
2539
+
adev->gfx.imu.funcs->start_imu(adev);
2540
+
}
2541
+
2542
+
/* disable gpa mode in backdoor loading */
2543
+
gfx_v12_1_xcc_disable_gpa_mode(adev, i);
2544
+
}
2545
+
}
2546
+
}
2547
+
2548
+
if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
2549
+
(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
2550
+
r = gfx_v12_1_wait_for_rlc_autoload_complete(adev);
2551
+
if (r) {
2552
+
dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
2553
+
return r;
2554
+
}
2555
+
}
2556
+
2557
+
adev->gfx.is_poweron = true;
2558
+
2559
+
if (get_gb_addr_config(adev))
2560
+
DRM_WARN("Invalid gb_addr_config !\n");
2561
+
2562
+
if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
2563
+
gfx_v12_1_config_gfx_rs64(adev);
2564
+
2565
+
r = gfx_v12_1_gfxhub_enable(adev);
2566
+
if (r)
2567
+
return r;
2568
+
2569
+
if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT ||
2570
+
adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) &&
2571
+
(amdgpu_dpm == 1)) {
2572
+
/**
2573
+
* For gfx 12, rlc firmware loading relies on smu firmware is
2574
+
* loaded firstly, so in direct type, it has to load smc ucode
2575
+
* here before rlc.
2576
+
*/
2577
+
if (!(adev->flags & AMD_IS_APU)) {
2578
+
r = amdgpu_pm_load_smu_firmware(adev, NULL);
2579
+
if (r)
2580
+
return r;
2581
+
}
2582
+
}
2583
+
2584
+
gfx_v12_1_init_golden_registers(adev);
2585
+
2586
+
gfx_v12_1_constants_init(adev);
2587
+
2588
+
if (adev->nbio.funcs->gc_doorbell_init)
2589
+
adev->nbio.funcs->gc_doorbell_init(adev);
2590
+
2591
+
r = gfx_v12_1_rlc_resume(adev);
2592
+
if (r)
2593
+
return r;
2594
+
2595
+
/*
2596
+
* init golden registers and rlc resume may override some registers,
2597
+
* reconfig them here
2598
+
*/
2599
+
gfx_v12_1_tcp_harvest(adev);
2600
+
2601
+
r = gfx_v12_1_cp_resume(adev);
2602
+
if (r)
2603
+
return r;
2604
+
2605
+
return r;
2606
+
}
2607
+
2608
+
static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev,
2609
+
int xcc_id)
2610
+
{
2611
+
uint32_t tmp;
2612
+
2613
+
if (!adev->no_hw_access) {
2614
+
if (amdgpu_gfx_disable_kcq(adev, xcc_id))
2615
+
DRM_ERROR("KCQ disable failed\n");
2616
+
2617
+
amdgpu_mes_kiq_hw_fini(adev, xcc_id);
2618
+
}
2619
+
2620
+
if (amdgpu_sriov_vf(adev)) {
2621
+
/* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
2622
+
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
2623
+
tmp &= 0xffffff00;
2624
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
2625
+
}
2626
+
gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
2627
+
gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
2628
+
}
2629
+
2630
+
static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
2631
+
{
2632
+
struct amdgpu_device *adev = ip_block->adev;
2633
+
int i, num_xcc;
2634
+
2635
+
amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
2636
+
amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
2637
+
2638
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2639
+
for (i = 0; i < num_xcc; i++) {
2640
+
gfx_v12_1_xcc_fini(adev, i);
2641
+
}
2642
+
2643
+
adev->gfxhub.funcs->gart_disable(adev);
2644
+
2645
+
adev->gfx.is_poweron = false;
2646
+
2647
+
return 0;
2648
+
}
2649
+
2650
+
static int gfx_v12_1_suspend(struct amdgpu_ip_block *ip_block)
2651
+
{
2652
+
return gfx_v12_1_hw_fini(ip_block);
2653
+
}
2654
+
2655
+
static int gfx_v12_1_resume(struct amdgpu_ip_block *ip_block)
2656
+
{
2657
+
return gfx_v12_1_hw_init(ip_block);
2658
+
}
2659
+
2660
+
static bool gfx_v12_1_is_idle(struct amdgpu_ip_block *ip_block)
2661
+
{
2662
+
struct amdgpu_device *adev = ip_block->adev;
2663
+
int i, num_xcc;
2664
+
2665
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2666
+
for (i = 0; i < num_xcc; i++) {
2667
+
if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i),
2668
+
regGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE))
2669
+
return false;
2670
+
}
2671
+
return true;
2672
+
}
2673
+
2674
+
static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
2675
+
{
2676
+
unsigned i;
2677
+
struct amdgpu_device *adev = ip_block->adev;
2678
+
2679
+
for (i = 0; i < adev->usec_timeout; i++) {
2680
+
if (gfx_v12_1_is_idle(ip_block))
2681
+
return 0;
2682
+
udelay(1);
2683
+
}
2684
+
return -ETIMEDOUT;
2685
+
}
2686
+
2687
+
static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev)
2688
+
{
2689
+
uint64_t clock = 0;
2690
+
2691
+
if (adev->smuio.funcs &&
2692
+
adev->smuio.funcs->get_gpu_clock_counter)
2693
+
clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
2694
+
else
2695
+
dev_warn(adev->dev, "query gpu clock counter is not supported\n");
2696
+
2697
+
return clock;
2698
+
}
2699
+
2700
+
static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
2701
+
{
2702
+
struct amdgpu_device *adev = ip_block->adev;
2703
+
2704
+
adev->gfx.funcs = &gfx_v12_1_gfx_funcs;
2705
+
2706
+
adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
2707
+
AMDGPU_MAX_COMPUTE_RINGS);
2708
+
2709
+
gfx_v12_1_set_kiq_pm4_funcs(adev);
2710
+
gfx_v12_1_set_ring_funcs(adev);
2711
+
gfx_v12_1_set_irq_funcs(adev);
2712
+
gfx_v12_1_set_rlc_funcs(adev);
2713
+
gfx_v12_1_set_mqd_funcs(adev);
2714
+
gfx_v12_1_set_imu_funcs(adev);
2715
+
2716
+
gfx_v12_1_init_rlcg_reg_access_ctrl(adev);
2717
+
2718
+
return gfx_v12_1_init_microcode(adev);
2719
+
}
2720
+
2721
+
static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block)
2722
+
{
2723
+
struct amdgpu_device *adev = ip_block->adev;
2724
+
int r;
2725
+
2726
+
r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
2727
+
if (r)
2728
+
return r;
2729
+
2730
+
r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
2731
+
if (r)
2732
+
return r;
2733
+
2734
+
return 0;
2735
+
}
2736
+
2737
+
static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev)
2738
+
{
2739
+
uint32_t rlc_cntl;
2740
+
2741
+
/* if RLC is not enabled, do nothing */
2742
+
rlc_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL);
2743
+
return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
2744
+
}
2745
+
2746
+
static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev,
2747
+
int xcc_id)
2748
+
{
2749
+
uint32_t data;
2750
+
unsigned i;
2751
+
2752
+
data = RLC_SAFE_MODE__CMD_MASK;
2753
+
data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
2754
+
2755
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
2756
+
2757
+
/* wait for RLC_SAFE_MODE */
2758
+
for (i = 0; i < adev->usec_timeout; i++) {
2759
+
if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2760
+
regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
2761
+
break;
2762
+
udelay(1);
2763
+
}
2764
+
}
2765
+
2766
+
static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev,
2767
+
int xcc_id)
2768
+
{
2769
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2770
+
regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
2771
+
}
2772
+
2773
+
static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
2774
+
bool enable)
2775
+
{
2776
+
int i, num_xcc;
2777
+
2778
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2779
+
for (i = 0; i < num_xcc; i++)
2780
+
gfx_v12_1_xcc_update_perf_clk(adev, enable, i);
2781
+
}
2782
+
2783
+
static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev,
2784
+
int xcc_id,
2785
+
struct amdgpu_ring *ring,
2786
+
unsigned vmid)
2787
+
{
2788
+
u32 reg, data;
2789
+
2790
+
reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
2791
+
if (amdgpu_sriov_is_pp_one_vf(adev))
2792
+
data = RREG32_NO_KIQ(reg);
2793
+
else
2794
+
data = RREG32(reg);
2795
+
2796
+
data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
2797
+
data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
2798
+
2799
+
if (amdgpu_sriov_is_pp_one_vf(adev))
2800
+
WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
2801
+
else
2802
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
2803
+
2804
+
if (ring
2805
+
&& amdgpu_sriov_is_pp_one_vf(adev)
2806
+
&& ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
2807
+
|| (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
2808
+
uint32_t reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
2809
+
amdgpu_ring_emit_wreg(ring, reg, data);
2810
+
}
2811
+
}
2812
+
2813
+
static const struct amdgpu_rlc_funcs gfx_v12_1_rlc_funcs = {
2814
+
.is_rlc_enabled = gfx_v12_1_is_rlc_enabled,
2815
+
.set_safe_mode = gfx_v12_1_xcc_set_safe_mode,
2816
+
.unset_safe_mode = gfx_v12_1_xcc_unset_safe_mode,
2817
+
.init = gfx_v12_1_rlc_init,
2818
+
.get_csb_size = gfx_v12_1_get_csb_size,
2819
+
.get_csb_buffer = gfx_v12_1_get_csb_buffer,
2820
+
.resume = gfx_v12_1_rlc_resume,
2821
+
.stop = gfx_v12_1_rlc_stop,
2822
+
.reset = gfx_v12_1_rlc_reset,
2823
+
.start = gfx_v12_1_rlc_start,
2824
+
.update_spm_vmid = gfx_v12_1_update_spm_vmid,
2825
+
};
2826
+
2827
+
#if 0
2828
+
static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
2829
+
{
2830
+
/* TODO */
2831
+
}
2832
+
2833
+
static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
2834
+
{
2835
+
/* TODO */
2836
+
}
2837
+
#endif
2838
+
2839
+
static int gfx_v12_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
2840
+
enum amd_powergating_state state)
2841
+
{
2842
+
struct amdgpu_device *adev = ip_block->adev;
2843
+
bool enable = (state == AMD_PG_STATE_GATE);
2844
+
2845
+
if (amdgpu_sriov_vf(adev))
2846
+
return 0;
2847
+
2848
+
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2849
+
case IP_VERSION(12, 1, 0):
2850
+
amdgpu_gfx_off_ctrl(adev, enable);
2851
+
break;
2852
+
default:
2853
+
break;
2854
+
}
2855
+
2856
+
return 0;
2857
+
}
2858
+
2859
+
static void gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2860
+
bool enable, int xcc_id)
2861
+
{
2862
+
uint32_t def, data;
2863
+
2864
+
if (!(adev->cg_flags &
2865
+
(AMD_CG_SUPPORT_GFX_CGCG |
2866
+
AMD_CG_SUPPORT_GFX_CGLS |
2867
+
AMD_CG_SUPPORT_GFX_3D_CGCG |
2868
+
AMD_CG_SUPPORT_GFX_3D_CGLS)))
2869
+
return;
2870
+
2871
+
if (enable) {
2872
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2873
+
regRLC_CGTT_MGCG_OVERRIDE);
2874
+
2875
+
/* unset CGCG override */
2876
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
2877
+
data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
2878
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2879
+
data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2880
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
2881
+
adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
2882
+
data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
2883
+
2884
+
/* update CGCG override bits */
2885
+
if (def != data)
2886
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2887
+
regRLC_CGTT_MGCG_OVERRIDE, data);
2888
+
2889
+
/* enable cgcg FSM(0x0000363F) */
2890
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2891
+
2892
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
2893
+
data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
2894
+
data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2895
+
RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2896
+
}
2897
+
2898
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
2899
+
data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
2900
+
data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2901
+
RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2902
+
}
2903
+
2904
+
if (def != data)
2905
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2906
+
regRLC_CGCG_CGLS_CTRL, data);
2907
+
2908
+
/* set IDLE_POLL_COUNT(0x00900100) */
2909
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
2910
+
2911
+
data &= ~CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK;
2912
+
data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2913
+
data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2914
+
(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2915
+
2916
+
if (def != data)
2917
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
2918
+
2919
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL);
2920
+
data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
2921
+
data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
2922
+
data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
2923
+
data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
2924
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL, data);
2925
+
} else {
2926
+
/* Program RLC_CGCG_CGLS_CTRL */
2927
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2928
+
2929
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
2930
+
data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2931
+
2932
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2933
+
data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2934
+
2935
+
if (def != data)
2936
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2937
+
}
2938
+
}
2939
+
2940
+
static void gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
2941
+
bool enable, int xcc_id)
2942
+
{
2943
+
uint32_t data, def;
2944
+
if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
2945
+
return;
2946
+
2947
+
/* It is disabled by HW by default */
2948
+
if (enable) {
2949
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
2950
+
/* 1 - RLC_CGTT_MGCG_OVERRIDE */
2951
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2952
+
2953
+
data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2954
+
RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2955
+
RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
2956
+
2957
+
if (def != data)
2958
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2959
+
}
2960
+
} else {
2961
+
if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
2962
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2963
+
2964
+
data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2965
+
RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2966
+
RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
2967
+
2968
+
if (def != data)
2969
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2970
+
}
2971
+
}
2972
+
}
2973
+
2974
+
static void gfx_v12_1_xcc_update_repeater_fgcg(struct amdgpu_device *adev,
2975
+
bool enable, int xcc_id)
2976
+
{
2977
+
uint32_t def, data;
2978
+
2979
+
if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
2980
+
return;
2981
+
2982
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2983
+
2984
+
if (enable)
2985
+
data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
2986
+
RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK);
2987
+
else
2988
+
data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
2989
+
RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK;
2990
+
2991
+
if (def != data)
2992
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2993
+
}
2994
+
2995
+
static void gfx_v12_1_xcc_update_sram_fgcg(struct amdgpu_device *adev,
2996
+
bool enable, int xcc_id)
2997
+
{
2998
+
uint32_t def, data;
2999
+
3000
+
if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
3001
+
return;
3002
+
3003
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3004
+
3005
+
if (enable)
3006
+
data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3007
+
else
3008
+
data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3009
+
3010
+
if (def != data)
3011
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3012
+
}
3013
+
3014
+
static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
3015
+
bool enable, int xcc_id)
3016
+
{
3017
+
uint32_t def, data;
3018
+
3019
+
if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
3020
+
return;
3021
+
3022
+
def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3023
+
3024
+
if (enable)
3025
+
data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3026
+
else
3027
+
data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3028
+
3029
+
if (def != data)
3030
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3031
+
}
3032
+
3033
+
static int gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
3034
+
bool enable, int xcc_id)
3035
+
{
3036
+
amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
3037
+
3038
+
gfx_v12_1_xcc_update_coarse_grain_clock_gating(adev, enable, xcc_id);
3039
+
3040
+
gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id);
3041
+
3042
+
gfx_v12_1_xcc_update_repeater_fgcg(adev, enable, xcc_id);
3043
+
3044
+
gfx_v12_1_xcc_update_sram_fgcg(adev, enable, xcc_id);
3045
+
3046
+
gfx_v12_1_xcc_update_perf_clk(adev, enable, xcc_id);
3047
+
3048
+
if (adev->cg_flags &
3049
+
(AMD_CG_SUPPORT_GFX_MGCG |
3050
+
AMD_CG_SUPPORT_GFX_CGLS |
3051
+
AMD_CG_SUPPORT_GFX_CGCG |
3052
+
AMD_CG_SUPPORT_GFX_3D_CGCG |
3053
+
AMD_CG_SUPPORT_GFX_3D_CGLS))
3054
+
gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, enable, xcc_id);
3055
+
3056
+
amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
3057
+
3058
+
return 0;
3059
+
}
3060
+
3061
+
static int gfx_v12_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
3062
+
enum amd_clockgating_state state)
3063
+
{
3064
+
struct amdgpu_device *adev = ip_block->adev;
3065
+
int i, num_xcc;
3066
+
3067
+
if (amdgpu_sriov_vf(adev))
3068
+
return 0;
3069
+
3070
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3071
+
switch (adev->ip_versions[GC_HWIP][0]) {
3072
+
case IP_VERSION(12, 1, 0):
3073
+
for (i = 0; i < num_xcc; i++)
3074
+
gfx_v12_1_xcc_update_gfx_clock_gating(adev,
3075
+
state == AMD_CG_STATE_GATE, i);
3076
+
break;
3077
+
default:
3078
+
break;
3079
+
}
3080
+
3081
+
return 0;
3082
+
}
3083
+
3084
+
static void gfx_v12_1_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
3085
+
{
3086
+
struct amdgpu_device *adev = ip_block->adev;
3087
+
int data;
3088
+
3089
+
/* AMD_CG_SUPPORT_GFX_MGCG */
3090
+
data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE);
3091
+
if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
3092
+
*flags |= AMD_CG_SUPPORT_GFX_MGCG;
3093
+
3094
+
/* AMD_CG_SUPPORT_REPEATER_FGCG */
3095
+
if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
3096
+
*flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
3097
+
3098
+
/* AMD_CG_SUPPORT_GFX_FGCG */
3099
+
if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
3100
+
*flags |= AMD_CG_SUPPORT_GFX_FGCG;
3101
+
3102
+
/* AMD_CG_SUPPORT_GFX_PERF_CLK */
3103
+
if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
3104
+
*flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
3105
+
3106
+
/* AMD_CG_SUPPORT_GFX_CGCG */
3107
+
data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL);
3108
+
if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
3109
+
*flags |= AMD_CG_SUPPORT_GFX_CGCG;
3110
+
3111
+
/* AMD_CG_SUPPORT_GFX_CGLS */
3112
+
if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
3113
+
*flags |= AMD_CG_SUPPORT_GFX_CGLS;
3114
+
}
3115
+
3116
+
static u64 gfx_v12_1_ring_get_rptr_compute(struct amdgpu_ring *ring)
3117
+
{
3118
+
/* gfx12 hardware is 32bit rptr */
3119
+
return *(uint32_t *)ring->rptr_cpu_addr;
3120
+
}
3121
+
3122
+
static u64 gfx_v12_1_ring_get_wptr_compute(struct amdgpu_ring *ring)
3123
+
{
3124
+
u64 wptr;
3125
+
3126
+
/* XXX check if swapping is necessary on BE */
3127
+
if (ring->use_doorbell)
3128
+
wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
3129
+
else
3130
+
BUG();
3131
+
return wptr;
3132
+
}
3133
+
3134
+
static void gfx_v12_1_ring_set_wptr_compute(struct amdgpu_ring *ring)
3135
+
{
3136
+
struct amdgpu_device *adev = ring->adev;
3137
+
3138
+
/* XXX check if swapping is necessary on BE */
3139
+
if (ring->use_doorbell) {
3140
+
atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
3141
+
ring->wptr);
3142
+
WDOORBELL64(ring->doorbell_index, ring->wptr);
3143
+
} else {
3144
+
BUG(); /* only DOORBELL method supported on gfx12 now */
3145
+
}
3146
+
}
3147
+
3148
+
static void gfx_v12_1_ring_emit_hdp_flush(struct amdgpu_ring *ring)
3149
+
{
3150
+
struct amdgpu_device *adev = ring->adev;
3151
+
u32 ref_and_mask, reg_mem_engine;
3152
+
const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
3153
+
3154
+
if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3155
+
switch (ring->me) {
3156
+
case 1:
3157
+
ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
3158
+
break;
3159
+
case 2:
3160
+
ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
3161
+
break;
3162
+
default:
3163
+
return;
3164
+
}
3165
+
reg_mem_engine = 0;
3166
+
} else {
3167
+
ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
3168
+
reg_mem_engine = 1; /* pfp */
3169
+
}
3170
+
3171
+
gfx_v12_1_wait_reg_mem(ring, reg_mem_engine, 0, 1,
3172
+
adev->nbio.funcs->get_hdp_flush_req_offset(adev),
3173
+
adev->nbio.funcs->get_hdp_flush_done_offset(adev),
3174
+
ref_and_mask, ref_and_mask, 0x20);
3175
+
}
3176
+
3177
+
static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring,
3178
+
struct amdgpu_job *job,
3179
+
struct amdgpu_ib *ib,
3180
+
uint32_t flags)
3181
+
{
3182
+
unsigned vmid = AMDGPU_JOB_GET_VMID(job);
3183
+
u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
3184
+
3185
+
/* Currently, there is a high possibility to get wave ID mismatch
3186
+
* between ME and GDS, leading to a hw deadlock, because ME generates
3187
+
* different wave IDs than the GDS expects. This situation happens
3188
+
* randomly when at least 5 compute pipes use GDS ordered append.
3189
+
* The wave IDs generated by ME are also wrong after suspend/resume.
3190
+
* Those are probably bugs somewhere else in the kernel driver.
3191
+
*
3192
+
* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
3193
+
* GDS to 0 for this ring (me/pipe).
3194
+
*/
3195
+
if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
3196
+
amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3197
+
amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
3198
+
}
3199
+
3200
+
amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3201
+
BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
3202
+
amdgpu_ring_write(ring,
3203
+
#ifdef __BIG_ENDIAN
3204
+
(2 << 0) |
3205
+
#endif
3206
+
lower_32_bits(ib->gpu_addr));
3207
+
amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3208
+
amdgpu_ring_write(ring, control);
3209
+
}
3210
+
3211
+
static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
3212
+
u64 seq, unsigned flags)
3213
+
{
3214
+
bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
3215
+
bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
3216
+
3217
+
/* RELEASE_MEM - flush caches, send int */
3218
+
amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
3219
+
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ(1) |
3220
+
PACKET3_RELEASE_MEM_GCR_GLV_WB |
3221
+
PACKET3_RELEASE_MEM_GCR_GL2_WB |
3222
+
PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(2) |
3223
+
PACKET3_RELEASE_MEM_TEMPORAL(3) |
3224
+
PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3225
+
PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
3226
+
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
3227
+
PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
3228
+
3229
+
/*
3230
+
* the address should be Qword aligned if 64bit write, Dword
3231
+
* aligned if only send 32bit data low (discard data high)
3232
+
*/
3233
+
if (write64bit)
3234
+
BUG_ON(addr & 0x7);
3235
+
else
3236
+
BUG_ON(addr & 0x3);
3237
+
amdgpu_ring_write(ring, lower_32_bits(addr));
3238
+
amdgpu_ring_write(ring, upper_32_bits(addr));
3239
+
amdgpu_ring_write(ring, lower_32_bits(seq));
3240
+
amdgpu_ring_write(ring, upper_32_bits(seq));
3241
+
amdgpu_ring_write(ring, 0);
3242
+
}
3243
+
3244
+
static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3245
+
{
3246
+
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3247
+
uint32_t seq = ring->fence_drv.sync_seq;
3248
+
uint64_t addr = ring->fence_drv.gpu_addr;
3249
+
3250
+
gfx_v12_1_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
3251
+
upper_32_bits(addr), seq, 0xffffffff, 4);
3252
+
}
3253
+
3254
+
static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
3255
+
uint16_t pasid, uint32_t flush_type,
3256
+
bool all_hub, uint8_t dst_sel)
3257
+
{
3258
+
amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
3259
+
amdgpu_ring_write(ring,
3260
+
PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
3261
+
PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
3262
+
PACKET3_INVALIDATE_TLBS_PASID(pasid) |
3263
+
PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
3264
+
}
3265
+
3266
+
static void gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
3267
+
unsigned vmid, uint64_t pd_addr)
3268
+
{
3269
+
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3270
+
3271
+
/* compute doesn't have PFP */
3272
+
if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
3273
+
/* sync PFP to ME, otherwise we might get invalid PFP reads */
3274
+
amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3275
+
amdgpu_ring_write(ring, 0x0);
3276
+
}
3277
+
}
3278
+
3279
+
static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
3280
+
u64 seq, unsigned int flags)
3281
+
{
3282
+
struct amdgpu_device *adev = ring->adev;
3283
+
3284
+
/* we only allocate 32bit for each seq wb address */
3285
+
BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
3286
+
3287
+
/* write fence seq to the "addr" */
3288
+
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3289
+
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3290
+
WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
3291
+
amdgpu_ring_write(ring, lower_32_bits(addr));
3292
+
amdgpu_ring_write(ring, upper_32_bits(addr));
3293
+
amdgpu_ring_write(ring, lower_32_bits(seq));
3294
+
3295
+
if (flags & AMDGPU_FENCE_FLAG_INT) {
3296
+
/* set register to trigger INT */
3297
+
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3298
+
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3299
+
WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
3300
+
amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
3301
+
amdgpu_ring_write(ring, 0);
3302
+
amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
3303
+
}
3304
+
}
3305
+
3306
+
static void gfx_v12_1_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
3307
+
uint32_t reg_val_offs)
3308
+
{
3309
+
struct amdgpu_device *adev = ring->adev;
3310
+
3311
+
amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3312
+
amdgpu_ring_write(ring, 0 | /* src: register*/
3313
+
(5 << 8) | /* dst: memory */
3314
+
(1 << 20)); /* write confirm */
3315
+
amdgpu_ring_write(ring, reg);
3316
+
amdgpu_ring_write(ring, 0);
3317
+
amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3318
+
reg_val_offs * 4));
3319
+
amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3320
+
reg_val_offs * 4));
3321
+
}
3322
+
3323
+
static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
3324
+
uint32_t reg,
3325
+
uint32_t val)
3326
+
{
3327
+
uint32_t cmd = 0;
3328
+
3329
+
switch (ring->funcs->type) {
3330
+
case AMDGPU_RING_TYPE_KIQ:
3331
+
cmd = (1 << 16); /* no inc addr */
3332
+
break;
3333
+
default:
3334
+
cmd = WR_CONFIRM;
3335
+
break;
3336
+
}
3337
+
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3338
+
amdgpu_ring_write(ring, cmd);
3339
+
amdgpu_ring_write(ring, reg);
3340
+
amdgpu_ring_write(ring, 0);
3341
+
amdgpu_ring_write(ring, val);
3342
+
}
3343
+
3344
+
static void gfx_v12_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
3345
+
uint32_t val, uint32_t mask)
3346
+
{
3347
+
gfx_v12_1_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
3348
+
}
3349
+
3350
+
static void gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
3351
+
uint32_t reg0, uint32_t reg1,
3352
+
uint32_t ref, uint32_t mask)
3353
+
{
3354
+
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3355
+
3356
+
gfx_v12_1_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
3357
+
ref, mask, 0x20);
3358
+
}
3359
+
3360
+
static void gfx_v12_1_xcc_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
3361
+
int me, int pipe,
3362
+
enum amdgpu_interrupt_state state,
3363
+
int xcc_id)
3364
+
{
3365
+
u32 mec_int_cntl, mec_int_cntl_reg;
3366
+
3367
+
/*
3368
+
* amdgpu controls only the first MEC. That's why this function only
3369
+
* handles the setting of interrupts for this specific MEC. All other
3370
+
* pipes' interrupts are set by amdkfd.
3371
+
*/
3372
+
3373
+
if (me == 1) {
3374
+
switch (pipe) {
3375
+
case 0:
3376
+
mec_int_cntl_reg = SOC15_REG_OFFSET(
3377
+
GC, GET_INST(GC, xcc_id),
3378
+
regCP_ME1_PIPE0_INT_CNTL);
3379
+
break;
3380
+
case 1:
3381
+
mec_int_cntl_reg = SOC15_REG_OFFSET(
3382
+
GC, GET_INST(GC, xcc_id),
3383
+
regCP_ME1_PIPE1_INT_CNTL);
3384
+
break;
3385
+
case 2:
3386
+
mec_int_cntl_reg = SOC15_REG_OFFSET(
3387
+
GC, GET_INST(GC, xcc_id),
3388
+
regCP_ME1_PIPE2_INT_CNTL);
3389
+
break;
3390
+
case 3:
3391
+
mec_int_cntl_reg = SOC15_REG_OFFSET(
3392
+
GC, GET_INST(GC, xcc_id),
3393
+
regCP_ME1_PIPE3_INT_CNTL);
3394
+
break;
3395
+
default:
3396
+
DRM_DEBUG("invalid pipe %d\n", pipe);
3397
+
return;
3398
+
}
3399
+
} else {
3400
+
DRM_DEBUG("invalid me %d\n", me);
3401
+
return;
3402
+
}
3403
+
3404
+
switch (state) {
3405
+
case AMDGPU_IRQ_STATE_DISABLE:
3406
+
mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3407
+
mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3408
+
TIME_STAMP_INT_ENABLE, 0);
3409
+
mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3410
+
GENERIC0_INT_ENABLE, 0);
3411
+
WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3412
+
break;
3413
+
case AMDGPU_IRQ_STATE_ENABLE:
3414
+
mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3415
+
mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3416
+
TIME_STAMP_INT_ENABLE, 1);
3417
+
mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3418
+
GENERIC0_INT_ENABLE, 1);
3419
+
WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3420
+
break;
3421
+
default:
3422
+
break;
3423
+
}
3424
+
}
3425
+
3426
+
static int gfx_v12_1_set_eop_interrupt_state(struct amdgpu_device *adev,
3427
+
struct amdgpu_irq_src *src,
3428
+
unsigned type,
3429
+
enum amdgpu_interrupt_state state)
3430
+
{
3431
+
int i, num_xcc;
3432
+
3433
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3434
+
for (i = 0; i < num_xcc; i++) {
3435
+
switch (type) {
3436
+
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3437
+
gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3438
+
adev, 1, 0, state, i);
3439
+
break;
3440
+
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3441
+
gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3442
+
adev, 1, 1, state, i);
3443
+
break;
3444
+
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
3445
+
gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3446
+
adev, 1, 2, state, i);
3447
+
break;
3448
+
case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
3449
+
gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3450
+
adev, 1, 3, state, i);
3451
+
break;
3452
+
default:
3453
+
break;
3454
+
}
3455
+
}
3456
+
3457
+
return 0;
3458
+
}
3459
+
3460
+
static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
3461
+
struct amdgpu_irq_src *source,
3462
+
struct amdgpu_iv_entry *entry)
3463
+
{
3464
+
int i;
3465
+
u8 me_id, pipe_id, queue_id;
3466
+
struct amdgpu_ring *ring;
3467
+
uint32_t mes_queue_id = entry->src_data[0];
3468
+
3469
+
DRM_DEBUG("IH: CP EOP\n");
3470
+
3471
+
if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
3472
+
struct amdgpu_mes_queue *queue;
3473
+
3474
+
mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
3475
+
3476
+
spin_lock(&adev->mes.queue_id_lock);
3477
+
queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
3478
+
if (queue) {
3479
+
DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
3480
+
amdgpu_fence_process(queue->ring);
3481
+
}
3482
+
spin_unlock(&adev->mes.queue_id_lock);
3483
+
} else {
3484
+
me_id = (entry->ring_id & 0x0c) >> 2;
3485
+
pipe_id = (entry->ring_id & 0x03) >> 0;
3486
+
queue_id = (entry->ring_id & 0x70) >> 4;
3487
+
3488
+
switch (me_id) {
3489
+
case 0:
3490
+
if (pipe_id == 0)
3491
+
amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
3492
+
else
3493
+
amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
3494
+
break;
3495
+
case 1:
3496
+
case 2:
3497
+
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3498
+
ring = &adev->gfx.compute_ring[i];
3499
+
/* Per-queue interrupt is supported for MEC starting from VI.
3500
+
* The interrupt can only be enabled/disabled per pipe instead
3501
+
* of per queue.
3502
+
*/
3503
+
if ((ring->me == me_id) &&
3504
+
(ring->pipe == pipe_id) &&
3505
+
(ring->queue == queue_id))
3506
+
amdgpu_fence_process(ring);
3507
+
}
3508
+
break;
3509
+
}
3510
+
}
3511
+
3512
+
return 0;
3513
+
}
3514
+
3515
+
static int gfx_v12_1_set_priv_reg_fault_state(struct amdgpu_device *adev,
3516
+
struct amdgpu_irq_src *source,
3517
+
unsigned type,
3518
+
enum amdgpu_interrupt_state state)
3519
+
{
3520
+
int i, num_xcc;
3521
+
3522
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3523
+
switch (state) {
3524
+
case AMDGPU_IRQ_STATE_DISABLE:
3525
+
case AMDGPU_IRQ_STATE_ENABLE:
3526
+
for (i = 0; i < num_xcc; i++)
3527
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3528
+
PRIV_REG_INT_ENABLE,
3529
+
state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3530
+
break;
3531
+
default:
3532
+
break;
3533
+
}
3534
+
3535
+
return 0;
3536
+
}
3537
+
3538
+
static int gfx_v12_1_set_priv_inst_fault_state(struct amdgpu_device *adev,
3539
+
struct amdgpu_irq_src *source,
3540
+
unsigned type,
3541
+
enum amdgpu_interrupt_state state)
3542
+
{
3543
+
int i, num_xcc;
3544
+
3545
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3546
+
switch (state) {
3547
+
case AMDGPU_IRQ_STATE_DISABLE:
3548
+
case AMDGPU_IRQ_STATE_ENABLE:
3549
+
for (i = 0; i < num_xcc; i++)
3550
+
WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3551
+
PRIV_INSTR_INT_ENABLE,
3552
+
state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3553
+
break;
3554
+
default:
3555
+
break;
3556
+
}
3557
+
3558
+
return 0;
3559
+
}
3560
+
3561
+
static void gfx_v12_1_handle_priv_fault(struct amdgpu_device *adev,
3562
+
struct amdgpu_iv_entry *entry)
3563
+
{
3564
+
u8 me_id, pipe_id, queue_id;
3565
+
struct amdgpu_ring *ring;
3566
+
int i;
3567
+
3568
+
me_id = (entry->ring_id & 0x0c) >> 2;
3569
+
pipe_id = (entry->ring_id & 0x03) >> 0;
3570
+
queue_id = (entry->ring_id & 0x70) >> 4;
3571
+
3572
+
switch (me_id) {
3573
+
case 0:
3574
+
for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3575
+
ring = &adev->gfx.gfx_ring[i];
3576
+
/* we only enabled 1 gfx queue per pipe for now */
3577
+
if (ring->me == me_id && ring->pipe == pipe_id)
3578
+
drm_sched_fault(&ring->sched);
3579
+
}
3580
+
break;
3581
+
case 1:
3582
+
case 2:
3583
+
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3584
+
ring = &adev->gfx.compute_ring[i];
3585
+
if (ring->me == me_id && ring->pipe == pipe_id &&
3586
+
ring->queue == queue_id)
3587
+
drm_sched_fault(&ring->sched);
3588
+
}
3589
+
break;
3590
+
default:
3591
+
BUG();
3592
+
break;
3593
+
}
3594
+
}
3595
+
3596
+
static int gfx_v12_1_priv_reg_irq(struct amdgpu_device *adev,
3597
+
struct amdgpu_irq_src *source,
3598
+
struct amdgpu_iv_entry *entry)
3599
+
{
3600
+
DRM_ERROR("Illegal register access in command stream\n");
3601
+
gfx_v12_1_handle_priv_fault(adev, entry);
3602
+
return 0;
3603
+
}
3604
+
3605
+
static int gfx_v12_1_priv_inst_irq(struct amdgpu_device *adev,
3606
+
struct amdgpu_irq_src *source,
3607
+
struct amdgpu_iv_entry *entry)
3608
+
{
3609
+
DRM_ERROR("Illegal instruction in command stream\n");
3610
+
gfx_v12_1_handle_priv_fault(adev, entry);
3611
+
return 0;
3612
+
}
3613
+
3614
+
static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
3615
+
{
3616
+
const unsigned int gcr_cntl =
3617
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
3618
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
3619
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
3620
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
3621
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1) |
3622
+
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(2);
3623
+
3624
+
/* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
3625
+
amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
3626
+
amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
3627
+
amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
3628
+
amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
3629
+
amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
3630
+
amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
3631
+
amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
3632
+
amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
3633
+
}
3634
+
3635
+
static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
3636
+
.name = "gfx_v12_1",
3637
+
.early_init = gfx_v12_1_early_init,
3638
+
.late_init = gfx_v12_1_late_init,
3639
+
.sw_init = gfx_v12_1_sw_init,
3640
+
.sw_fini = gfx_v12_1_sw_fini,
3641
+
.hw_init = gfx_v12_1_hw_init,
3642
+
.hw_fini = gfx_v12_1_hw_fini,
3643
+
.suspend = gfx_v12_1_suspend,
3644
+
.resume = gfx_v12_1_resume,
3645
+
.is_idle = gfx_v12_1_is_idle,
3646
+
.wait_for_idle = gfx_v12_1_wait_for_idle,
3647
+
.set_clockgating_state = gfx_v12_1_set_clockgating_state,
3648
+
.set_powergating_state = gfx_v12_1_set_powergating_state,
3649
+
.get_clockgating_state = gfx_v12_1_get_clockgating_state,
3650
+
};
3651
+
3652
+
static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = {
3653
+
.type = AMDGPU_RING_TYPE_COMPUTE,
3654
+
.align_mask = 0xff,
3655
+
.nop = PACKET3(PACKET3_NOP, 0x3FFF),
3656
+
.support_64bit_ptrs = true,
3657
+
.get_rptr = gfx_v12_1_ring_get_rptr_compute,
3658
+
.get_wptr = gfx_v12_1_ring_get_wptr_compute,
3659
+
.set_wptr = gfx_v12_1_ring_set_wptr_compute,
3660
+
.emit_frame_size =
3661
+
7 + /* gfx_v12_1_ring_emit_hdp_flush */
3662
+
5 + /* hdp invalidate */
3663
+
7 + /* gfx_v12_1_ring_emit_pipeline_sync */
3664
+
SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
3665
+
SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
3666
+
2 + /* gfx_v12_1_ring_emit_vm_flush */
3667
+
8 + 8 + 8 + /* gfx_v12_1_ring_emit_fence x3 for user fence, vm fence */
3668
+
8, /* gfx_v12_1_emit_mem_sync */
3669
+
.emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */
3670
+
.emit_ib = gfx_v12_1_ring_emit_ib_compute,
3671
+
.emit_fence = gfx_v12_1_ring_emit_fence,
3672
+
.emit_pipeline_sync = gfx_v12_1_ring_emit_pipeline_sync,
3673
+
.emit_vm_flush = gfx_v12_1_ring_emit_vm_flush,
3674
+
.emit_hdp_flush = gfx_v12_1_ring_emit_hdp_flush,
3675
+
.test_ring = gfx_v12_1_ring_test_ring,
3676
+
.test_ib = gfx_v12_1_ring_test_ib,
3677
+
.insert_nop = amdgpu_ring_insert_nop,
3678
+
.pad_ib = amdgpu_ring_generic_pad_ib,
3679
+
.emit_wreg = gfx_v12_1_ring_emit_wreg,
3680
+
.emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
3681
+
.emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
3682
+
.emit_mem_sync = gfx_v12_1_emit_mem_sync,
3683
+
};
3684
+
3685
+
static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_kiq = {
3686
+
.type = AMDGPU_RING_TYPE_KIQ,
3687
+
.align_mask = 0xff,
3688
+
.nop = PACKET3(PACKET3_NOP, 0x3FFF),
3689
+
.support_64bit_ptrs = true,
3690
+
.get_rptr = gfx_v12_1_ring_get_rptr_compute,
3691
+
.get_wptr = gfx_v12_1_ring_get_wptr_compute,
3692
+
.set_wptr = gfx_v12_1_ring_set_wptr_compute,
3693
+
.emit_frame_size =
3694
+
7 + /* gfx_v12_1_ring_emit_hdp_flush */
3695
+
5 + /*hdp invalidate */
3696
+
7 + /* gfx_v12_1_ring_emit_pipeline_sync */
3697
+
SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
3698
+
SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
3699
+
2 + /* gfx_v12_1_ring_emit_vm_flush */
3700
+
8 + 8 + 8, /* gfx_v12_1_ring_emit_fence_kiq x3 for user fence, vm fence */
3701
+
.emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */
3702
+
.emit_ib = gfx_v12_1_ring_emit_ib_compute,
3703
+
.emit_fence = gfx_v12_1_ring_emit_fence_kiq,
3704
+
.test_ring = gfx_v12_1_ring_test_ring,
3705
+
.test_ib = gfx_v12_1_ring_test_ib,
3706
+
.insert_nop = amdgpu_ring_insert_nop,
3707
+
.pad_ib = amdgpu_ring_generic_pad_ib,
3708
+
.emit_rreg = gfx_v12_1_ring_emit_rreg,
3709
+
.emit_wreg = gfx_v12_1_ring_emit_wreg,
3710
+
.emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
3711
+
.emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
3712
+
};
3713
+
3714
+
static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev)
3715
+
{
3716
+
int i, j, num_xcc;
3717
+
3718
+
num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3719
+
for (i = 0; i < num_xcc; i++) {
3720
+
adev->gfx.kiq[i].ring.funcs = &gfx_v12_1_ring_funcs_kiq;
3721
+
3722
+
for (j = 0; j < adev->gfx.num_compute_rings; j++)
3723
+
adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs =
3724
+
&gfx_v12_1_ring_funcs_compute;
3725
+
}
3726
+
}
3727
+
3728
+
static const struct amdgpu_irq_src_funcs gfx_v12_1_eop_irq_funcs = {
3729
+
.set = gfx_v12_1_set_eop_interrupt_state,
3730
+
.process = gfx_v12_1_eop_irq,
3731
+
};
3732
+
3733
+
static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_reg_irq_funcs = {
3734
+
.set = gfx_v12_1_set_priv_reg_fault_state,
3735
+
.process = gfx_v12_1_priv_reg_irq,
3736
+
};
3737
+
3738
+
static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_inst_irq_funcs = {
3739
+
.set = gfx_v12_1_set_priv_inst_fault_state,
3740
+
.process = gfx_v12_1_priv_inst_irq,
3741
+
};
3742
+
3743
+
static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
3744
+
{
3745
+
adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
3746
+
adev->gfx.eop_irq.funcs = &gfx_v12_1_eop_irq_funcs;
3747
+
3748
+
adev->gfx.priv_reg_irq.num_types = 1;
3749
+
adev->gfx.priv_reg_irq.funcs = &gfx_v12_1_priv_reg_irq_funcs;
3750
+
3751
+
adev->gfx.priv_inst_irq.num_types = 1;
3752
+
adev->gfx.priv_inst_irq.funcs = &gfx_v12_1_priv_inst_irq_funcs;
3753
+
}
3754
+
3755
+
static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev)
3756
+
{
3757
+
#if 0
3758
+
if (adev->flags & AMD_IS_APU)
3759
+
adev->gfx.imu.mode = MISSION_MODE;
3760
+
else
3761
+
adev->gfx.imu.mode = DEBUG_MODE;
3762
+
3763
+
adev->gfx.imu.funcs = &gfx_v12_0_imu_funcs;
3764
+
#endif
3765
+
}
3766
+
3767
+
static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev)
3768
+
{
3769
+
adev->gfx.rlc.funcs = &gfx_v12_1_rlc_funcs;
3770
+
}
3771
+
3772
+
static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev)
3773
+
{
3774
+
/* set compute eng mqd */
3775
+
adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
3776
+
sizeof(struct v12_1_compute_mqd);
3777
+
adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
3778
+
gfx_v12_1_compute_mqd_init;
3779
+
}
3780
+
3781
+
static void gfx_v12_1_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
3782
+
u32 bitmap, int xcc_id)
3783
+
{
3784
+
u32 data;
3785
+
3786
+
if (!bitmap)
3787
+
return;
3788
+
3789
+
data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
3790
+
data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
3791
+
3792
+
WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data);
3793
+
}
3794
+
3795
+
static u32 gfx_v12_1_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev,
3796
+
int xcc_id)
3797
+
{
3798
+
u32 data, wgp_bitmask;
3799
+
data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG);
3800
+
data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG);
3801
+
3802
+
data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
3803
+
data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
3804
+
3805
+
wgp_bitmask =
3806
+
amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
3807
+
3808
+
return (~data) & wgp_bitmask;
3809
+
}
3810
+
3811
+
static u32 gfx_v12_1_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev,
3812
+
int xcc_id)
3813
+
{
3814
+
u32 wgp_idx, wgp_active_bitmap;
3815
+
u32 cu_bitmap_per_wgp, cu_active_bitmap;
3816
+
3817
+
wgp_active_bitmap = gfx_v12_1_get_wgp_active_bitmap_per_sh(adev, xcc_id);
3818
+
cu_active_bitmap = 0;
3819
+
3820
+
for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
3821
+
/* if there is one WGP enabled, it means 2 CUs will be enabled */
3822
+
cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
3823
+
if (wgp_active_bitmap & (1 << wgp_idx))
3824
+
cu_active_bitmap |= cu_bitmap_per_wgp;
3825
+
}
3826
+
3827
+
return cu_active_bitmap;
3828
+
}
3829
+
3830
+
static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
3831
+
struct amdgpu_cu_info *cu_info)
3832
+
{
3833
+
int i, j, k, counter, xcc_id, active_cu_number = 0;
3834
+
u32 mask, bitmap;
3835
+
unsigned disable_masks[8 * 2];
3836
+
3837
+
if (!adev || !cu_info)
3838
+
return -EINVAL;
3839
+
3840
+
amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
3841
+
3842
+
mutex_lock(&adev->grbm_idx_mutex);
3843
+
for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
3844
+
for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3845
+
for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3846
+
bitmap = i * adev->gfx.config.max_sh_per_se + j;
3847
+
if (!((gfx_v12_1_get_sa_active_bitmap(adev, xcc_id) >> bitmap) & 1))
3848
+
continue;
3849
+
mask = 1;
3850
+
counter = 0;
3851
+
gfx_v12_1_xcc_select_se_sh(adev, i, j, 0xffffffff, xcc_id);
3852
+
if (i < 8 && j < 2)
3853
+
gfx_v12_1_set_user_wgp_inactive_bitmap_per_sh(
3854
+
adev, disable_masks[i * 2 + j], xcc_id);
3855
+
bitmap = gfx_v12_1_get_cu_active_bitmap_per_sh(adev, xcc_id);
3856
+
3857
+
/**
3858
+
* GFX12 could support more than 4 SEs, while the bitmap
3859
+
* in cu_info struct is 4x4 and ioctl interface struct
3860
+
* drm_amdgpu_info_device should keep stable.
3861
+
* So we use last two columns of bitmap to store cu mask for
3862
+
* SEs 4 to 7, the layout of the bitmap is as below:
3863
+
* SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
3864
+
* SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
3865
+
* SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
3866
+
* SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
3867
+
* SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
3868
+
* SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
3869
+
* SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
3870
+
* SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
3871
+
*/
3872
+
cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
3873
+
3874
+
for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
3875
+
if (bitmap & mask)
3876
+
counter++;
3877
+
3878
+
mask <<= 1;
3879
+
}
3880
+
active_cu_number += counter;
3881
+
}
3882
+
}
3883
+
gfx_v12_1_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, xcc_id);
3884
+
}
3885
+
mutex_unlock(&adev->grbm_idx_mutex);
3886
+
3887
+
cu_info->number = active_cu_number;
3888
+
cu_info->simd_per_cu = NUM_SIMD_PER_CU;
3889
+
cu_info->lds_size = 320;
3890
+
3891
+
return 0;
3892
+
}
3893
+
3894
+
const struct amdgpu_ip_block_version gfx_v12_1_ip_block = {
3895
+
.type = AMD_IP_BLOCK_TYPE_GFX,
3896
+
.major = 12,
3897
+
.minor = 1,
3898
+
.rev = 0,
3899
+
.funcs = &gfx_v12_1_ip_funcs,
3900
+
};
3901
+
3902
+
static int gfx_v12_1_xcp_resume(void *handle, uint32_t inst_mask)
3903
+
{
3904
+
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3905
+
uint32_t tmp_mask;
3906
+
int i, r;
3907
+
3908
+
/* TODO : Initialize golden regs */
3909
+
/* gfx_v12_1_init_golden_registers(adev); */
3910
+
3911
+
tmp_mask = inst_mask;
3912
+
for_each_inst(i, tmp_mask)
3913
+
gfx_v12_1_xcc_constants_init(adev, i);
3914
+
3915
+
if (!amdgpu_sriov_vf(adev)) {
3916
+
tmp_mask = inst_mask;
3917
+
for_each_inst(i, tmp_mask) {
3918
+
r = gfx_v12_1_xcc_rlc_resume(adev, i);
3919
+
if (r)
3920
+
return r;
3921
+
}
3922
+
}
3923
+
3924
+
tmp_mask = inst_mask;
3925
+
for_each_inst(i, tmp_mask) {
3926
+
r = gfx_v12_1_xcc_cp_resume(adev, i);
3927
+
if (r)
3928
+
return r;
3929
+
}
3930
+
3931
+
return 0;
3932
+
}
3933
+
3934
+
static int gfx_v12_1_xcp_suspend(void *handle, uint32_t inst_mask)
3935
+
{
3936
+
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3937
+
int i;
3938
+
3939
+
for_each_inst(i, inst_mask)
3940
+
gfx_v12_1_xcc_fini(adev, i);
3941
+
3942
+
return 0;
3943
+
}
3944
+
3945
+
struct amdgpu_xcp_ip_funcs gfx_v12_1_xcp_funcs = {
3946
+
.suspend = &gfx_v12_1_xcp_suspend,
3947
+
.resume = &gfx_v12_1_xcp_resume
3948
+
};
+29
drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h
+29
drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h
···
1
+
/*
2
+
* Copyright 2025 Advanced Micro Devices, Inc.
3
+
*
4
+
* Permission is hereby granted, free of charge, to any person obtaining a
5
+
* copy of this software and associated documentation files (the "Software"),
6
+
* to deal in the Software without restriction, including without limitation
7
+
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
+
* and/or sell copies of the Software, and to permit persons to whom the
9
+
* Software is furnished to do so, subject to the following conditions:
10
+
*
11
+
* The above copyright notice and this permission notice shall be included in
12
+
* all copies or substantial portions of the Software.
13
+
*
14
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17
+
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
+
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
+
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
+
* OTHER DEALINGS IN THE SOFTWARE.
21
+
*
22
+
*/
23
+
24
+
#ifndef __GFX_V12_1_H__
25
+
#define __GFX_V12_1_H__
26
+
27
+
extern const struct amdgpu_ip_block_version gfx_v12_1_ip_block;
28
+
29
+
#endif