Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_gt_mcr.h"
7
8#include "regs/xe_gt_regs.h"
9#include "xe_assert.h"
10#include "xe_gt_printk.h"
11#include "xe_gt_topology.h"
12#include "xe_gt_types.h"
13#include "xe_guc_hwconfig.h"
14#include "xe_mmio.h"
15#include "xe_sriov.h"
16
17/**
18 * DOC: GT Multicast/Replicated (MCR) Register Support
19 *
20 * Some GT registers are designed as "multicast" or "replicated" registers:
21 * multiple instances of the same register share a single MMIO offset. MCR
22 * registers are generally used when the hardware needs to potentially track
23 * independent values of a register per hardware unit (e.g., per-subslice,
24 * per-L3bank, etc.). The specific types of replication that exist vary
25 * per-platform.
26 *
27 * MMIO accesses to MCR registers are controlled according to the settings
28 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
29 * registers can be done in either multicast (a single write updates all
30 * instances of the register to the same value) or unicast (a write updates only
31 * one specific instance) form. Reads of MCR registers always operate in a
32 * unicast manner regardless of how the multicast/unicast bit is set in
33 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is
34 * referred to as "steering."
35 *
36 * If MCR register operations are steered toward a hardware unit that is
37 * fused off or currently powered down due to power gating, the MMIO operation
38 * is "terminated" by the hardware. Terminated read operations will return a
39 * value of zero and terminated unicast write operations will be silently
40 * ignored. During device initialization, the goal of the various
41 * ``init_steering_*()`` functions is to apply the platform-specific rules for
42 * each MCR register type to identify a steering target that will select a
43 * non-terminated instance.
44 *
45 * MCR registers are not available on Virtual Function (VF).
46 */
47
48static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr)
49{
50 return reg_mcr.__reg;
51}
52
53enum {
54 MCR_OP_READ,
55 MCR_OP_WRITE
56};
57
58static const struct xe_mmio_range xelp_l3bank_steering_table[] = {
59 { 0x00B100, 0x00B3FF },
60 {},
61};
62
63static const struct xe_mmio_range xehp_l3bank_steering_table[] = {
64 { 0x008C80, 0x008CFF },
65 { 0x00B100, 0x00B3FF },
66 {},
67};
68
69/*
70 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
71 * are of a "GAM" subclass that has special rules and doesn't need to be
72 * included here.
73 */
74static const struct xe_mmio_range xehp_mslice_steering_table[] = {
75 { 0x00DD00, 0x00DDFF },
76 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
77 {},
78};
79
80static const struct xe_mmio_range xehp_lncf_steering_table[] = {
81 { 0x00B000, 0x00B0FF },
82 { 0x00D880, 0x00D8FF },
83 {},
84};
85
86/*
87 * We have several types of MCR registers where steering to (0,0) will always
88 * provide us with a non-terminated value. We'll stick them all in the same
89 * table for simplicity.
90 */
91static const struct xe_mmio_range xehpc_instance0_steering_table[] = {
92 { 0x004000, 0x004AFF }, /* HALF-BSLICE */
93 { 0x008800, 0x00887F }, /* CC */
94 { 0x008A80, 0x008AFF }, /* TILEPSMI */
95 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
96 { 0x00B100, 0x00B3FF }, /* L3BANK */
97 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
98 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
99 { 0x00DD00, 0x00DDFF }, /* BSLICE */
100 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
101 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
102 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
103 { 0x024180, 0x0241FF }, /* HALF-BSLICE */
104 {},
105};
106
107static const struct xe_mmio_range xelpg_instance0_steering_table[] = {
108 { 0x000B00, 0x000BFF }, /* SQIDI */
109 { 0x001000, 0x001FFF }, /* SQIDI */
110 { 0x004000, 0x0048FF }, /* GAM */
111 { 0x008700, 0x0087FF }, /* SQIDI */
112 { 0x00B000, 0x00B0FF }, /* NODE */
113 { 0x00C800, 0x00CFFF }, /* GAM */
114 { 0x00D880, 0x00D8FF }, /* NODE */
115 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
116 {},
117};
118
119static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {
120 { 0x00B100, 0x00B3FF },
121 {},
122};
123
124static const struct xe_mmio_range xelp_dss_steering_table[] = {
125 { 0x008150, 0x00815F },
126 { 0x009520, 0x00955F },
127 { 0x00DE80, 0x00E8FF },
128 { 0x024A00, 0x024A7F },
129 {},
130};
131
132/* DSS steering is used for GSLICE ranges as well */
133static const struct xe_mmio_range xehp_dss_steering_table[] = {
134 { 0x005200, 0x0052FF }, /* GSLICE */
135 { 0x005400, 0x007FFF }, /* GSLICE */
136 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
137 { 0x008D00, 0x008DFF }, /* DSS */
138 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
139 { 0x009680, 0x0096FF }, /* DSS */
140 { 0x00D800, 0x00D87F }, /* GSLICE */
141 { 0x00DC00, 0x00DCFF }, /* GSLICE */
142 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */
143 { 0x017000, 0x017FFF }, /* GSLICE */
144 { 0x024A00, 0x024A7F }, /* DSS */
145 {},
146};
147
148/* DSS steering is used for COMPUTE ranges as well */
149static const struct xe_mmio_range xehpc_dss_steering_table[] = {
150 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */
151 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */
152 { 0x009680, 0x0096FF }, /* DSS */
153 { 0x00DC00, 0x00DCFF }, /* COMPUTE */
154 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */
155 {},
156};
157
158/* DSS steering is used for SLICE ranges as well */
159static const struct xe_mmio_range xelpg_dss_steering_table[] = {
160 { 0x005200, 0x0052FF }, /* SLICE */
161 { 0x005500, 0x007FFF }, /* SLICE */
162 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
163 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
164 { 0x009680, 0x0096FF }, /* DSS */
165 { 0x00D800, 0x00D87F }, /* SLICE */
166 { 0x00DC00, 0x00DCFF }, /* SLICE */
167 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
168 {},
169};
170
171static const struct xe_mmio_range xe3p_xpc_xecore_steering_table[] = {
172 { 0x008140, 0x00817F }, /* SLICE, XeCore, SLICE */
173 { 0x009480, 0x00955F }, /* SLICE, XeCore */
174 { 0x00D800, 0x00D87F }, /* SLICE */
175 { 0x00DC00, 0x00E9FF }, /* SLICE, rsvd, XeCore, rsvd, XeCore, rsvd, XeCore */
176 { 0x013000, 0x0135FF }, /* XeCore, SLICE */
177 {},
178};
179
180static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {
181 { 0x393200, 0x39323F },
182 { 0x393400, 0x3934FF },
183 {},
184};
185
186static const struct xe_mmio_range dg2_implicit_steering_table[] = {
187 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */
188 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */
189 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */
190 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */
191 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */
192 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */
193 {},
194};
195
196static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {
197 { 0x005200, 0x0052FF }, /* SLICE */
198 { 0x005500, 0x007FFF }, /* SLICE */
199 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
200 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
201 { 0x009680, 0x0096FF }, /* DSS */
202 { 0x00D800, 0x00D87F }, /* SLICE */
203 { 0x00DC00, 0x00DCFF }, /* SLICE */
204 { 0x00DE00, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
205 { 0x00E980, 0x00E9FF }, /* SLICE */
206 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */
207 {},
208};
209
210static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {
211 { 0x000B00, 0x000BFF },
212 { 0x001000, 0x001FFF },
213 {},
214};
215
216static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {
217 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */
218 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */
219 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */
220 { 0x00C800, 0x00CFFF }, /* GAM */
221 { 0x00D880, 0x00D8FF }, /* NODE */
222 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */
223 { 0x00E900, 0x00E97F }, /* MEMPIPE */
224 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */
225 { 0x013400, 0x0135FF }, /* MEMPIPE */
226 {},
227};
228
229static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = {
230 { 0x388160, 0x38817F },
231 { 0x389480, 0x3894CF },
232 {},
233};
234
235static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = {
236 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */
237 { 0x384900, 0x384AFF }, /* GAM */
238 { 0x389560, 0x3895FF }, /* MEDIAINF */
239 { 0x38B600, 0x38B8FF }, /* L3BANK */
240 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */
241 { 0x38F000, 0x38F0FF }, /* GAM */
242 { 0x393C00, 0x393C7F }, /* MEDIAINF */
243 {},
244};
245
246static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = {
247 { 0x384000, 0x3841FF }, /* GAM */
248 { 0x384400, 0x3847DF }, /* GAM */
249 { 0x384900, 0x384AFF }, /* GAM */
250 { 0x389560, 0x3895FF }, /* MEDIAINF */
251 { 0x38B600, 0x38B8FF }, /* L3BANK */
252 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */
253 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, rsvd, GAM */
254 { 0x393C00, 0x393C7F }, /* MEDIAINF */
255 {},
256};
257
258/*
259 * Different "GAM" ranges have different rules; GAMWKRS, STLB, and GAMREQSTRM
260 * range subtypes need to be steered to (1,0), while all other GAM subtypes
261 * are steered to (0,0) and are included in the "INSTANCE0" table farther
262 * down.
263 */
264static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = {
265 { 0x004000, 0x004AFF }, /* GAMREQSTRM, rsvd, STLB, GAMWKRS, GAMREQSTRM */
266 { 0x00F100, 0x00FFFF }, /* GAMWKRS */
267 {},
268};
269
270static const struct xe_mmio_range xe3p_xpc_node_steering_table[] = {
271 { 0x00B000, 0x00B0FF },
272 { 0x00D880, 0x00D8FF },
273 {},
274};
275
276static const struct xe_mmio_range xe3p_xpc_instance0_steering_table[] = {
277 { 0x00B500, 0x00B6FF }, /* PSMI */
278 { 0x00C800, 0x00CFFF }, /* GAMCTRL */
279 { 0x00F000, 0x00F0FF }, /* GAMCTRL */
280 {},
281};
282
283static const struct xe_mmio_range xe3p_lpg_instance0_steering_table[] = {
284 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */
285 { 0x008700, 0x00887F }, /* NODE */
286 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */
287 { 0x00B500, 0x00B6FF }, /* PSMI */
288 { 0x00C800, 0x00CFFF }, /* GAM */
289 { 0x00D880, 0x00D8FF }, /* NODE */
290 { 0x00DD00, 0x00DD7F }, /* MEMPIPE */
291 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */
292 { 0x013400, 0x0135FF }, /* MEMPIPE */
293 {},
294};
295
296static void init_steering_l3bank(struct xe_gt *gt)
297{
298 struct xe_device *xe = gt_to_xe(gt);
299 struct xe_mmio *mmio = >->mmio;
300
301 if (GRAPHICS_VER(xe) >= 35) {
302 unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask);
303 const int banks_per_node = 4;
304 unsigned int node = first_bank / banks_per_node;
305
306 /* L3BANK ranges place node in grpID, bank in instanceid */
307 gt->steering[L3BANK].group_target = node;
308 gt->steering[L3BANK].instance_target = first_bank % banks_per_node;
309
310 /* NODE ranges split the node across grpid and instanceid */
311 gt->steering[NODE].group_target = node >> 1;
312 gt->steering[NODE].instance_target = node & 1;
313 } else if (GRAPHICS_VERx100(xe) >= 1270) {
314 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
315 xe_mmio_read32(mmio, MIRROR_FUSE3));
316 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,
317 xe_mmio_read32(mmio, XEHP_FUSE4));
318
319 /*
320 * Group selects mslice, instance selects bank within mslice.
321 * Bank 0 is always valid _except_ when the bank mask is 010b.
322 */
323 gt->steering[L3BANK].group_target = __ffs(mslice_mask);
324 gt->steering[L3BANK].instance_target =
325 bank_mask & BIT(0) ? 0 : 2;
326 } else if (xe->info.platform == XE_DG2) {
327 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
328 xe_mmio_read32(mmio, MIRROR_FUSE3));
329 u32 bank = __ffs(mslice_mask) * 8;
330
331 /*
332 * Like mslice registers, look for a valid mslice and steer to
333 * the first L3BANK of that quad. Access to the Nth L3 bank is
334 * split between the first bits of group and instance
335 */
336 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7;
337 gt->steering[L3BANK].instance_target = bank & 0x3;
338 } else {
339 u32 fuse = REG_FIELD_GET(L3BANK_MASK,
340 ~xe_mmio_read32(mmio, MIRROR_FUSE3));
341
342 gt->steering[L3BANK].group_target = 0; /* unused */
343 gt->steering[L3BANK].instance_target = __ffs(fuse);
344 }
345}
346
347static void init_steering_mslice(struct xe_gt *gt)
348{
349 u32 mask = REG_FIELD_GET(MEML3_EN_MASK,
350 xe_mmio_read32(>->mmio, MIRROR_FUSE3));
351
352 /*
353 * mslice registers are valid (not terminated) if either the meml3
354 * associated with the mslice is present, or at least one DSS associated
355 * with the mslice is present. There will always be at least one meml3
356 * so we can just use that to find a non-terminated mslice and ignore
357 * the DSS fusing.
358 */
359 gt->steering[MSLICE].group_target = __ffs(mask);
360 gt->steering[MSLICE].instance_target = 0; /* unused */
361
362 /*
363 * LNCF termination is also based on mslice presence, so we'll set
364 * it up here. Either LNCF within a non-terminated mslice will work,
365 * so we just always pick LNCF 0 here.
366 */
367 gt->steering[LNCF].group_target = __ffs(mask) << 1;
368 gt->steering[LNCF].instance_target = 0; /* unused */
369}
370
371static unsigned int dss_per_group(struct xe_gt *gt)
372{
373 struct xe_guc *guc = >->uc.guc;
374 u32 max_slices = 0, max_subslices = 0;
375 int ret;
376
377 /*
378 * Try to query the GuC's hwconfig table for the maximum number of
379 * slices and subslices. These don't reflect the platform's actual
380 * slice/DSS counts, just the physical layout by which we should
381 * determine the steering targets. On older platforms with older GuC
382 * firmware releases it's possible that these attributes may not be
383 * included in the table, so we can always fall back to the old
384 * hardcoded layouts.
385 */
386#define HWCONFIG_ATTR_MAX_SLICES 1
387#define HWCONFIG_ATTR_MAX_SUBSLICES 70
388
389 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES,
390 &max_slices);
391 if (ret < 0 || max_slices == 0)
392 goto fallback;
393
394 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES,
395 &max_subslices);
396 if (ret < 0 || max_subslices == 0)
397 goto fallback;
398
399 return DIV_ROUND_UP(max_subslices, max_slices);
400
401fallback:
402 /*
403 * Some older platforms don't have tables or don't have complete tables.
404 * Newer platforms should always have the required info.
405 */
406 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 &&
407 !gt_to_xe(gt)->info.force_execlist)
408 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n");
409
410 if (gt_to_xe(gt)->info.platform == XE_PVC)
411 return 8;
412 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
413 return 4;
414 else
415 return 6;
416}
417
418/**
419 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS
420 * @gt: GT structure
421 * @dss: DSS ID to obtain steering for
422 * @group: pointer to storage for steering group ID
423 * @instance: pointer to storage for steering instance ID
424 */
425void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance)
426{
427 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS);
428
429 *group = dss / gt->steering_dss_per_grp;
430 *instance = dss % gt->steering_dss_per_grp;
431}
432
433/**
434 * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering
435 * @gt: GT structure
436 * @group: steering group ID
437 * @instance: steering instance ID
438 *
439 * Return: the converted DSS id.
440 */
441u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance)
442{
443 return group * dss_per_group(gt) + instance;
444}
445
446static void init_steering_dss(struct xe_gt *gt)
447{
448 gt->steering_dss_per_grp = dss_per_group(gt);
449
450 xe_gt_mcr_get_dss_steering(gt,
451 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
452 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)),
453 >->steering[DSS].group_target,
454 >->steering[DSS].instance_target);
455}
456
457static void init_steering_oaddrm(struct xe_gt *gt)
458{
459 /*
460 * First instance is only terminated if the entire first media slice
461 * is absent (i.e., no VCS0 or VECS0).
462 */
463 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))
464 gt->steering[OADDRM].group_target = 0;
465 else
466 gt->steering[OADDRM].group_target = 1;
467
468 gt->steering[OADDRM].instance_target = 0; /* unused */
469}
470
471static void init_steering_sqidi_psmi(struct xe_gt *gt)
472{
473 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK,
474 xe_mmio_read32(>->mmio, MIRROR_FUSE3));
475 u32 select = __ffs(mask);
476
477 gt->steering[SQIDI_PSMI].group_target = select >> 1;
478 gt->steering[SQIDI_PSMI].instance_target = select & 0x1;
479}
480
481static void init_steering_gam1(struct xe_gt *gt)
482{
483 gt->steering[GAM1].group_target = 1;
484 gt->steering[GAM1].instance_target = 0;
485}
486
487static const struct {
488 const char *name;
489 void (*init)(struct xe_gt *gt);
490} xe_steering_types[] = {
491 [L3BANK] = { "L3BANK", init_steering_l3bank },
492 [NODE] = { "NODE", NULL }, /* initialized by l3bank init */
493 [MSLICE] = { "MSLICE", init_steering_mslice },
494 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */
495 [DSS] = { "DSS / XeCore", init_steering_dss },
496 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm },
497 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi },
498 [GAM1] = { "GAMWKRS / STLB / GAMREQSTRM", init_steering_gam1 },
499 [INSTANCE0] = { "INSTANCE 0", NULL },
500 [IMPLICIT_STEERING] = { "IMPLICIT", NULL },
501};
502
503/**
504 * xe_gt_mcr_init_early - Early initialization of the MCR support
505 * @gt: GT structure
506 *
507 * Perform early software only initialization of the MCR lock to allow
508 * the synchronization on accessing the STEER_SEMAPHORE register and
509 * use the xe_gt_mcr_multicast_write() function, plus the minimum
510 * safe MCR registers required for VRAM/CCS probing.
511 */
512void xe_gt_mcr_init_early(struct xe_gt *gt)
513{
514 struct xe_device *xe = gt_to_xe(gt);
515
516 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES);
517 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);
518
519 spin_lock_init(>->mcr_lock);
520
521 if (gt->info.type == XE_GT_TYPE_MEDIA) {
522 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
523
524 if (MEDIA_VER(xe) >= 30) {
525 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
526 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table;
527 } else if (MEDIA_VERx100(xe) >= 1301) {
528 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
529 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table;
530 } else {
531 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;
532 }
533 } else {
534 if (GRAPHICS_VERx100(xe) == 3511) {
535 gt->steering[DSS].ranges = xe3p_xpc_xecore_steering_table;
536 gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table;
537 gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table;
538 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
539 gt->steering[NODE].ranges = xe3p_xpc_node_steering_table;
540 } else if (GRAPHICS_VERx100(xe) >= 3510) {
541 gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
542 gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table;
543 } else if (GRAPHICS_VER(xe) >= 20) {
544 gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
545 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
546 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;
547 } else if (GRAPHICS_VERx100(xe) >= 1270) {
548 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
549 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
550 gt->steering[DSS].ranges = xelpg_dss_steering_table;
551 } else if (xe->info.platform == XE_PVC) {
552 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;
553 gt->steering[DSS].ranges = xehpc_dss_steering_table;
554 } else if (xe->info.platform == XE_DG2) {
555 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table;
556 gt->steering[MSLICE].ranges = xehp_mslice_steering_table;
557 gt->steering[LNCF].ranges = xehp_lncf_steering_table;
558 gt->steering[DSS].ranges = xehp_dss_steering_table;
559 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table;
560 } else {
561 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;
562 gt->steering[DSS].ranges = xelp_dss_steering_table;
563 }
564 }
565
566 /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */
567 gt->steering[INSTANCE0].initialized = true;
568}
569
570/**
571 * xe_gt_mcr_init - Normal initialization of the MCR support
572 * @gt: GT structure
573 *
574 * Perform normal initialization of the MCR for all usages.
575 */
576void xe_gt_mcr_init(struct xe_gt *gt)
577{
578 /* Select non-terminated steering target for each type */
579 for (int i = 0; i < NUM_STEERING_TYPES; i++) {
580 gt->steering[i].initialized = true;
581 if (gt->steering[i].ranges && xe_steering_types[i].init)
582 xe_steering_types[i].init(gt);
583 }
584}
585
586/**
587 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers
588 * @gt: GT structure
589 *
590 * Some register ranges don't need to have their steering control registers
591 * changed on each access - it's sufficient to set them once on initialization.
592 * This function sets those registers for each platform *
593 */
594void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)
595{
596 struct xe_device *xe = gt_to_xe(gt);
597
598 if (IS_SRIOV_VF(xe))
599 return;
600
601 if (xe->info.platform == XE_DG2) {
602 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) |
603 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2);
604
605 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val);
606 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val);
607 /*
608 * For GAM registers, all reads should be directed to instance 1
609 * (unicast reads against other instances are not allowed),
610 * and instance 1 is already the hardware's default steering
611 * target, which we never change
612 */
613 }
614}
615
616/*
617 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that
618 * will steer a register to a non-terminated instance
619 * @gt: GT structure
620 * @reg: register for which the steering is required
621 * @group: return variable for group steering
622 * @instance: return variable for instance steering
623 *
624 * This function returns a group/instance pair that is guaranteed to work for
625 * read steering of the given register. Note that a value will be returned even
626 * if the register is not replicated and therefore does not actually require
627 * steering.
628 *
629 * Returns true if the caller should steer to the @group/@instance values
630 * returned. Returns false if the caller need not perform any steering
631 */
632bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
633 struct xe_reg_mcr reg_mcr,
634 u8 *group, u8 *instance)
635{
636 const struct xe_reg reg = to_xe_reg(reg_mcr);
637 const struct xe_mmio_range *implicit_ranges;
638
639 for (int type = 0; type < IMPLICIT_STEERING; type++) {
640 if (!gt->steering[type].ranges)
641 continue;
642
643 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
644 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) {
645 drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized,
646 "Uninitialized usage of MCR register %s/%#x\n",
647 xe_steering_types[type].name, reg.addr);
648
649 *group = gt->steering[type].group_target;
650 *instance = gt->steering[type].instance_target;
651 return true;
652 }
653 }
654 }
655
656 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;
657 if (implicit_ranges)
658 for (int i = 0; implicit_ranges[i].end > 0; i++)
659 if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg))
660 return false;
661
662 /*
663 * Not found in a steering table and not a register with implicit
664 * steering. Just steer to 0/0 as a guess and raise a warning.
665 */
666 drm_WARN(>_to_xe(gt)->drm, true,
667 "Did not find MCR register %#x in any MCR steering table\n",
668 reg.addr);
669 *group = 0;
670 *instance = 0;
671
672 return true;
673}
674
675/*
676 * Obtain exclusive access to MCR steering. On MTL and beyond we also need
677 * to synchronize with external clients (e.g., firmware), so a semaphore
678 * register will also need to be taken.
679 */
680static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock)
681{
682 struct xe_device *xe = gt_to_xe(gt);
683 int ret = 0;
684
685 spin_lock(>->mcr_lock);
686
687 /*
688 * Starting with MTL we also need to grab a semaphore register
689 * to synchronize with external agents (e.g., firmware) that now
690 * shares the same steering control register. The semaphore is obtained
691 * when a read to the relevant register returns 1.
692 */
693 if (GRAPHICS_VERx100(xe) >= 1270)
694 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,
695 true);
696
697 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
698}
699
700static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock)
701{
702 /* Release hardware semaphore - this is done by writing 1 to the register */
703 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
704 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1);
705
706 spin_unlock(>->mcr_lock);
707}
708
709/*
710 * Access a register with specific MCR steering
711 *
712 * Caller needs to make sure the relevant forcewake wells are up.
713 */
714static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
715 u8 rw_flag, int group, int instance, u32 value)
716{
717 const struct xe_reg reg = to_xe_reg(reg_mcr);
718 struct xe_mmio *mmio = >->mmio;
719 struct xe_reg steer_reg;
720 u32 steer_val, val = 0;
721
722 lockdep_assert_held(>->mcr_lock);
723
724 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
725 steer_reg = MTL_MCR_SELECTOR;
726 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
727 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);
728 } else {
729 steer_reg = MCR_SELECTOR;
730 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) |
731 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance);
732 }
733
734 /*
735 * Always leave the hardware in multicast mode when doing reads and only
736 * change it to unicast mode when doing writes of a specific instance.
737 *
738 * The setting of the multicast/unicast bit usually wouldn't matter for
739 * read operations (which always return the value from a single register
740 * instance regardless of how that bit is set), but some platforms may
741 * have workarounds requiring us to remain in multicast mode for reads,
742 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so
743 * we'll just go ahead and do so on all platforms; we'll only clear the
744 * multicast bit from the mask when explicitly doing a write operation.
745 *
746 * No need to save old steering reg value.
747 */
748 if (rw_flag == MCR_OP_READ)
749 steer_val |= MCR_MULTICAST;
750
751 xe_mmio_write32(mmio, steer_reg, steer_val);
752
753 if (rw_flag == MCR_OP_READ)
754 val = xe_mmio_read32(mmio, reg);
755 else
756 xe_mmio_write32(mmio, reg, value);
757
758 /*
759 * If we turned off the multicast bit (during a write) we're required
760 * to turn it back on before finishing. The group and instance values
761 * don't matter since they'll be re-programmed on the next MCR
762 * operation.
763 */
764 if (rw_flag == MCR_OP_WRITE)
765 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST);
766
767 return val;
768}
769
770/**
771 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register
772 * @gt: GT structure
773 * @reg_mcr: register to read
774 *
775 * Reads a GT MCR register. The read will be steered to a non-terminated
776 * instance (i.e., one that isn't fused off or powered down by power gating).
777 * This function assumes the caller is already holding any necessary forcewake
778 * domains.
779 *
780 * Returns the value from a non-terminated instance of @reg.
781 */
782u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr)
783{
784 const struct xe_reg reg = to_xe_reg(reg_mcr);
785 u8 group, instance;
786 u32 val;
787 bool steer;
788
789 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
790
791 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr,
792 &group, &instance);
793
794 if (steer) {
795 mcr_lock(gt);
796 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ,
797 group, instance, 0);
798 mcr_unlock(gt);
799 } else {
800 val = xe_mmio_read32(>->mmio, reg);
801 }
802
803 return val;
804}
805
806/**
807 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register
808 * @gt: GT structure
809 * @reg_mcr: the MCR register to read
810 * @group: the MCR group
811 * @instance: the MCR instance
812 *
813 * Returns the value read from an MCR register after steering toward a specific
814 * group/instance.
815 */
816u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,
817 struct xe_reg_mcr reg_mcr,
818 int group, int instance)
819{
820 u32 val;
821
822 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
823
824 mcr_lock(gt);
825 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0);
826 mcr_unlock(gt);
827
828 return val;
829}
830
831/**
832 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register
833 * @gt: GT structure
834 * @reg_mcr: the MCR register to write
835 * @value: value to write
836 * @group: the MCR group
837 * @instance: the MCR instance
838 *
839 * Write an MCR register in unicast mode after steering toward a specific
840 * group/instance.
841 */
842void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
843 u32 value, int group, int instance)
844{
845 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
846
847 mcr_lock(gt);
848 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value);
849 mcr_unlock(gt);
850}
851
852/**
853 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register
854 * @gt: GT structure
855 * @reg_mcr: the MCR register to write
856 * @value: value to write
857 *
858 * Write an MCR register in multicast mode to update all instances.
859 */
860void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
861 u32 value)
862{
863 struct xe_reg reg = to_xe_reg(reg_mcr);
864
865 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
866
867 /*
868 * Synchronize with any unicast operations. Once we have exclusive
869 * access, the MULTICAST bit should already be set, so there's no need
870 * to touch the steering register.
871 */
872 mcr_lock(gt);
873 xe_mmio_write32(>->mmio, reg, value);
874 mcr_unlock(gt);
875}
876
877void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)
878{
879 for (int i = 0; i < NUM_STEERING_TYPES; i++) {
880 if (gt->steering[i].ranges) {
881 drm_printf(p, "%s steering: group=%#x, instance=%#x\n",
882 xe_steering_types[i].name,
883 gt->steering[i].group_target,
884 gt->steering[i].instance_target);
885 for (int j = 0; gt->steering[i].ranges[j].end; j++)
886 drm_printf(p, "\t0x%06x - 0x%06x\n",
887 gt->steering[i].ranges[j].start,
888 gt->steering[i].ranges[j].end);
889 }
890 }
891}