Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_mocs.h"
7
8#include "regs/xe_gt_regs.h"
9#include "xe_device.h"
10#include "xe_exec_queue.h"
11#include "xe_force_wake.h"
12#include "xe_gt.h"
13#include "xe_gt_mcr.h"
14#include "xe_gt_printk.h"
15#include "xe_mmio.h"
16#include "xe_platform_types.h"
17#include "xe_pm.h"
18#include "xe_sriov.h"
19
20#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
21#define mocs_dbg xe_gt_dbg
22#else
23__printf(2, 3)
24static inline void mocs_dbg(const struct xe_gt *gt,
25 const char *format, ...)
26{ /* noop */ }
27#endif
28
29enum {
30 HAS_GLOBAL_MOCS = BIT(0),
31 HAS_LNCF_MOCS = BIT(1),
32};
33
34struct xe_mocs_entry {
35 u32 control_value;
36 u16 l3cc_value;
37 u16 used;
38};
39
40struct xe_mocs_info;
41
42struct xe_mocs_ops {
43 void (*dump)(struct xe_mocs_info *mocs, unsigned int flags,
44 struct xe_gt *gt, struct drm_printer *p);
45};
46
47struct xe_mocs_info {
48 /*
49 * Size of the spec's suggested MOCS programming table. The list of
50 * table entries from the spec can potentially be smaller than the
51 * number of hardware registers used to program the MOCS table; in such
52 * cases the registers for the remaining indices will be programmed to
53 * match unused_entries_index.
54 */
55 unsigned int table_size;
56 /* Number of MOCS entries supported by the hardware */
57 unsigned int num_mocs_regs;
58 const struct xe_mocs_entry *table;
59 const struct xe_mocs_ops *ops;
60 u8 uc_index;
61 u8 wb_index;
62 u8 unused_entries_index;
63};
64
65/* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */
66#define IG_PAT REG_BIT(8)
67#define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4)
68#define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2)
69
70/* Helper defines */
71#define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */
72#define PVC_NUM_MOCS_ENTRIES 3
73#define MTL_NUM_MOCS_ENTRIES 16
74#define XE2_NUM_MOCS_ENTRIES 16
75
76/* (e)LLC caching options */
77/*
78 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
79 * the same as LE_UC
80 */
81#define LE_0_PAGETABLE LE_CACHEABILITY(0)
82#define LE_1_UC LE_CACHEABILITY(1)
83#define LE_2_WT LE_CACHEABILITY(2)
84#define LE_3_WB LE_CACHEABILITY(3)
85
86/* Target cache */
87#define LE_TC_0_PAGETABLE LE_TGT_CACHE(0)
88#define LE_TC_1_LLC LE_TGT_CACHE(1)
89#define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2)
90#define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3)
91
92/* L3 caching options */
93#define L3_0_DIRECT L3_CACHEABILITY(0)
94#define L3_1_UC L3_CACHEABILITY(1)
95#define L3_2_RESERVED L3_CACHEABILITY(2)
96#define L3_3_WB L3_CACHEABILITY(3)
97
98/* L4 caching options */
99#define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0)
100#define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1)
101#define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3)
102
103#define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0)
104/* XD: WB Transient Display */
105#define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1)
106#define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3)
107
108#define XE2_L3_CLOS_MASK REG_GENMASK(7, 6)
109
110#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
111 [__idx] = { \
112 .control_value = __control_value, \
113 .l3cc_value = __l3cc_value, \
114 .used = 1, \
115 }
116
117/*
118 * MOCS tables
119 *
120 * These are the MOCS tables that are programmed across all the rings.
121 * The control value is programmed to all the rings that support the
122 * MOCS registers. While the l3cc_values are only programmed to the
123 * LNCFCMOCS0 - LNCFCMOCS32 registers.
124 *
125 * These tables are intended to be kept reasonably consistent across
126 * HW platforms, and for ICL+, be identical across OSes. To achieve
127 * that, the list of entries is published as part of bspec.
128 *
129 * Entries not part of the following tables are undefined as far as userspace is
130 * concerned and shouldn't be relied upon. The last few entries are reserved by
131 * the hardware. They should be initialized according to bspec and never used.
132 *
133 * NOTE1: These tables are part of bspec and defined as part of the hardware
134 * interface. It is expected that, for specific hardware platform, existing
135 * entries will remain constant and the table will only be updated by adding new
136 * entries, filling unused positions.
137 *
138 * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These
139 * reserved entries should never be used. They may be changed to low performant
140 * variants with better coherency in the future if more entries are needed.
141 */
142
143static const struct xe_mocs_entry gen12_mocs_desc[] = {
144 /* Base - L3 + LLC */
145 MOCS_ENTRY(2,
146 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
147 L3_3_WB),
148 /* Base - Uncached */
149 MOCS_ENTRY(3,
150 LE_1_UC | LE_TC_1_LLC,
151 L3_1_UC),
152 /* Base - L3 */
153 MOCS_ENTRY(4,
154 LE_1_UC | LE_TC_1_LLC,
155 L3_3_WB),
156 /* Base - LLC */
157 MOCS_ENTRY(5,
158 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
159 L3_1_UC),
160 /* Age 0 - LLC */
161 MOCS_ENTRY(6,
162 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1),
163 L3_1_UC),
164 /* Age 0 - L3 + LLC */
165 MOCS_ENTRY(7,
166 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1),
167 L3_3_WB),
168 /* Age: Don't Chg. - LLC */
169 MOCS_ENTRY(8,
170 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2),
171 L3_1_UC),
172 /* Age: Don't Chg. - L3 + LLC */
173 MOCS_ENTRY(9,
174 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2),
175 L3_3_WB),
176 /* No AOM - LLC */
177 MOCS_ENTRY(10,
178 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1),
179 L3_1_UC),
180 /* No AOM - L3 + LLC */
181 MOCS_ENTRY(11,
182 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1),
183 L3_3_WB),
184 /* No AOM; Age 0 - LLC */
185 MOCS_ENTRY(12,
186 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1),
187 L3_1_UC),
188 /* No AOM; Age 0 - L3 + LLC */
189 MOCS_ENTRY(13,
190 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1),
191 L3_3_WB),
192 /* No AOM; Age:DC - LLC */
193 MOCS_ENTRY(14,
194 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1),
195 L3_1_UC),
196 /* No AOM; Age:DC - L3 + LLC */
197 MOCS_ENTRY(15,
198 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1),
199 L3_3_WB),
200 /* Self-Snoop - L3 + LLC */
201 MOCS_ENTRY(18,
202 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3),
203 L3_3_WB),
204 /* Skip Caching - L3 + LLC(12.5%) */
205 MOCS_ENTRY(19,
206 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7),
207 L3_3_WB),
208 /* Skip Caching - L3 + LLC(25%) */
209 MOCS_ENTRY(20,
210 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3),
211 L3_3_WB),
212 /* Skip Caching - L3 + LLC(50%) */
213 MOCS_ENTRY(21,
214 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1),
215 L3_3_WB),
216 /* Skip Caching - L3 + LLC(75%) */
217 MOCS_ENTRY(22,
218 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3),
219 L3_3_WB),
220 /* Skip Caching - L3 + LLC(87.5%) */
221 MOCS_ENTRY(23,
222 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7),
223 L3_3_WB),
224 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
225 MOCS_ENTRY(48,
226 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
227 L3_3_WB),
228 /* Implicitly enable L1 - HDC:L1 + L3 */
229 MOCS_ENTRY(49,
230 LE_1_UC | LE_TC_1_LLC,
231 L3_3_WB),
232 /* Implicitly enable L1 - HDC:L1 + LLC */
233 MOCS_ENTRY(50,
234 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
235 L3_1_UC),
236 /* Implicitly enable L1 - HDC:L1 */
237 MOCS_ENTRY(51,
238 LE_1_UC | LE_TC_1_LLC,
239 L3_1_UC),
240 /* HW Special Case (CCS) */
241 MOCS_ENTRY(60,
242 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
243 L3_1_UC),
244 /* HW Special Case (Displayable) */
245 MOCS_ENTRY(61,
246 LE_1_UC | LE_TC_1_LLC,
247 L3_3_WB),
248 /* HW Reserved - SW program but never use */
249 MOCS_ENTRY(62,
250 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
251 L3_1_UC),
252 /* HW Reserved - SW program but never use */
253 MOCS_ENTRY(63,
254 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
255 L3_1_UC)
256};
257
258static bool regs_are_mcr(struct xe_gt *gt)
259{
260 struct xe_device *xe = gt_to_xe(gt);
261
262 if (xe_gt_is_media_type(gt))
263 return MEDIA_VER(xe) >= 20;
264 else
265 return GRAPHICS_VERx100(xe) >= 1250;
266}
267
268static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p)
269{
270 unsigned int i, j;
271 u32 reg_val;
272
273 drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n");
274
275 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {
276 if (regs_are_mcr(gt))
277 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));
278 else
279 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i));
280
281 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",
282 j++,
283 !!(reg_val & L3_ESC_MASK),
284 REG_FIELD_GET(L3_SCC_MASK, reg_val),
285 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),
286 reg_val);
287
288 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",
289 j,
290 !!(reg_val & L3_UPPER_IDX_ESC_MASK),
291 REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val),
292 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),
293 reg_val);
294 }
295}
296
297static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags,
298 struct xe_gt *gt, struct drm_printer *p)
299{
300 unsigned int i;
301 u32 reg_val;
302
303 if (flags & HAS_GLOBAL_MOCS) {
304 drm_printf(p, "Global mocs table configuration:\n");
305 drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n");
306
307 for (i = 0; i < info->num_mocs_regs; i++) {
308 if (regs_are_mcr(gt))
309 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));
310 else
311 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i));
312
313 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n",
314 i,
315 REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val),
316 REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val),
317 REG_FIELD_GET(LE_LRUM_MASK, reg_val),
318 !!(reg_val & LE_AOM_MASK),
319 !!(reg_val & LE_RSC_MASK),
320 REG_FIELD_GET(LE_SCC_MASK, reg_val),
321 REG_FIELD_GET(LE_PFM_MASK, reg_val),
322 !!(reg_val & LE_SCF_MASK),
323 REG_FIELD_GET(LE_COS_MASK, reg_val),
324 REG_FIELD_GET(LE_SSE_MASK, reg_val),
325 reg_val);
326 }
327 }
328
329 xelp_lncf_dump(info, gt, p);
330}
331
332static const struct xe_mocs_ops xelp_mocs_ops = {
333 .dump = xelp_mocs_dump,
334};
335
336static const struct xe_mocs_entry dg1_mocs_desc[] = {
337 /* UC */
338 MOCS_ENTRY(1, 0, L3_1_UC),
339 /* WB - L3 */
340 MOCS_ENTRY(5, 0, L3_3_WB),
341 /* WB - L3 50% */
342 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB),
343 /* WB - L3 25% */
344 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB),
345 /* WB - L3 12.5% */
346 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB),
347
348 /* HDC:L1 + L3 */
349 MOCS_ENTRY(48, 0, L3_3_WB),
350 /* HDC:L1 */
351 MOCS_ENTRY(49, 0, L3_1_UC),
352
353 /* HW Reserved */
354 MOCS_ENTRY(60, 0, L3_1_UC),
355 MOCS_ENTRY(61, 0, L3_1_UC),
356 MOCS_ENTRY(62, 0, L3_1_UC),
357 MOCS_ENTRY(63, 0, L3_1_UC),
358};
359
360static const struct xe_mocs_entry dg2_mocs_desc[] = {
361 /* UC - Coherent; GO:L3 */
362 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)),
363 /* UC - Coherent; GO:Memory */
364 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
365 /* UC - Non-Coherent; GO:Memory */
366 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)),
367
368 /* WB - LC */
369 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)),
370};
371
372static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags,
373 struct xe_gt *gt, struct drm_printer *p)
374{
375 unsigned int i, j;
376 u32 reg_val;
377
378 drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n");
379
380 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {
381 if (regs_are_mcr(gt))
382 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));
383 else
384 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i));
385
386 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",
387 j++,
388 !!(reg_val & L3_LKUP_MASK),
389 !!(reg_val & L3_GLBGO_MASK),
390 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),
391 reg_val);
392
393 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n",
394 j,
395 !!(reg_val & L3_UPPER_LKUP_MASK),
396 !!(reg_val & L3_UPPER_GLBGO_MASK),
397 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),
398 reg_val);
399 }
400}
401
402static const struct xe_mocs_ops xehp_mocs_ops = {
403 .dump = xehp_lncf_dump,
404};
405
406static const struct xe_mocs_entry pvc_mocs_desc[] = {
407 /* Error */
408 MOCS_ENTRY(0, 0, L3_3_WB),
409
410 /* UC */
411 MOCS_ENTRY(1, 0, L3_1_UC),
412
413 /* WB */
414 MOCS_ENTRY(2, 0, L3_3_WB),
415};
416
417static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt,
418 struct drm_printer *p)
419{
420 unsigned int i, j;
421 u32 reg_val;
422
423 drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n");
424
425 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) {
426 if (regs_are_mcr(gt))
427 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));
428 else
429 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i));
430
431 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n",
432 j++,
433 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val),
434 reg_val);
435
436 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n",
437 j,
438 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val),
439 reg_val);
440 }
441}
442
443static const struct xe_mocs_ops pvc_mocs_ops = {
444 .dump = pvc_mocs_dump,
445};
446
447static const struct xe_mocs_entry mtl_mocs_desc[] = {
448 /* Error - Reserved for Non-Use */
449 MOCS_ENTRY(0,
450 0,
451 L3_LKUP(1) | L3_3_WB),
452 /* Cached - L3 + L4 */
453 MOCS_ENTRY(1,
454 IG_PAT,
455 L3_LKUP(1) | L3_3_WB),
456 /* L4 - GO:L3 */
457 MOCS_ENTRY(2,
458 IG_PAT,
459 L3_LKUP(1) | L3_1_UC),
460 /* Uncached - GO:L3 */
461 MOCS_ENTRY(3,
462 IG_PAT | L4_3_UC,
463 L3_LKUP(1) | L3_1_UC),
464 /* L4 - GO:Mem */
465 MOCS_ENTRY(4,
466 IG_PAT,
467 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC),
468 /* Uncached - GO:Mem */
469 MOCS_ENTRY(5,
470 IG_PAT | L4_3_UC,
471 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC),
472 /* L4 - L3:NoLKUP; GO:L3 */
473 MOCS_ENTRY(6,
474 IG_PAT,
475 L3_1_UC),
476 /* Uncached - L3:NoLKUP; GO:L3 */
477 MOCS_ENTRY(7,
478 IG_PAT | L4_3_UC,
479 L3_1_UC),
480 /* L4 - L3:NoLKUP; GO:Mem */
481 MOCS_ENTRY(8,
482 IG_PAT,
483 L3_GLBGO(1) | L3_1_UC),
484 /* Uncached - L3:NoLKUP; GO:Mem */
485 MOCS_ENTRY(9,
486 IG_PAT | L4_3_UC,
487 L3_GLBGO(1) | L3_1_UC),
488 /* Display - L3; L4:WT */
489 MOCS_ENTRY(14,
490 IG_PAT | L4_1_WT,
491 L3_LKUP(1) | L3_3_WB),
492 /* CCS - Non-Displayable */
493 MOCS_ENTRY(15,
494 IG_PAT,
495 L3_GLBGO(1) | L3_1_UC),
496};
497
498static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags,
499 struct xe_gt *gt, struct drm_printer *p)
500{
501 unsigned int i;
502 u32 reg_val;
503
504 drm_printf(p, "Global mocs table configuration:\n");
505 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n");
506
507 for (i = 0; i < info->num_mocs_regs; i++) {
508 if (regs_are_mcr(gt))
509 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));
510 else
511 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i));
512
513 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n",
514 i,
515 !!(reg_val & IG_PAT),
516 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val),
517 reg_val);
518 }
519
520 /* MTL lncf mocs table pattern is similar to that of xehp */
521 xehp_lncf_dump(info, flags, gt, p);
522}
523
524static const struct xe_mocs_ops mtl_mocs_ops = {
525 .dump = mtl_mocs_dump,
526};
527
528static const struct xe_mocs_entry xe2_mocs_table[] = {
529 /* Defer to PAT */
530 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0),
531 /* Cached L3, Uncached L4 */
532 MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0),
533 /* Uncached L3, Cached L4 */
534 MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0),
535 /* Uncached L3 + L4 */
536 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0),
537 /* Cached L3 + L4 */
538 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0),
539};
540
541static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags,
542 struct xe_gt *gt, struct drm_printer *p)
543{
544 unsigned int i;
545 u32 reg_val;
546
547 drm_printf(p, "Global mocs table configuration:\n");
548 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n");
549
550 for (i = 0; i < info->num_mocs_regs; i++) {
551 if (regs_are_mcr(gt))
552 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));
553 else
554 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i));
555
556 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n",
557 i,
558 !!(reg_val & IG_PAT),
559 REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val),
560 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val),
561 reg_val);
562 }
563}
564
565static const struct xe_mocs_ops xe2_mocs_ops = {
566 .dump = xe2_mocs_dump,
567};
568
569/*
570 * Note that the "L3" and "L4" register fields actually control the L2 and L3
571 * caches respectively on this platform.
572 */
573static const struct xe_mocs_entry xe3p_xpc_mocs_table[] = {
574 /* Defer to PAT */
575 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0),
576 /* UC */
577 MOCS_ENTRY(1, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0),
578 /* L2 */
579 MOCS_ENTRY(2, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0),
580 /* L3 */
581 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0),
582 /* L2 + L3 */
583 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0),
584};
585
586static unsigned int get_mocs_settings(struct xe_device *xe,
587 struct xe_mocs_info *info)
588{
589 unsigned int flags = 0;
590
591 memset(info, 0, sizeof(struct xe_mocs_info));
592
593 switch (xe->info.platform) {
594 case XE_CRESCENTISLAND:
595 info->ops = &xe2_mocs_ops;
596 info->table_size = ARRAY_SIZE(xe3p_xpc_mocs_table);
597 info->table = xe3p_xpc_mocs_table;
598 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES;
599 info->uc_index = 1;
600 info->wb_index = 4;
601 info->unused_entries_index = 4;
602 break;
603 case XE_NOVALAKE_P:
604 case XE_NOVALAKE_S:
605 case XE_PANTHERLAKE:
606 case XE_LUNARLAKE:
607 case XE_BATTLEMAGE:
608 info->ops = &xe2_mocs_ops;
609 info->table_size = ARRAY_SIZE(xe2_mocs_table);
610 info->table = xe2_mocs_table;
611 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES;
612 info->uc_index = 3;
613 info->wb_index = 4;
614 info->unused_entries_index = 4;
615 break;
616 case XE_PVC:
617 info->ops = &pvc_mocs_ops;
618 info->table_size = ARRAY_SIZE(pvc_mocs_desc);
619 info->table = pvc_mocs_desc;
620 info->num_mocs_regs = PVC_NUM_MOCS_ENTRIES;
621 info->uc_index = 1;
622 info->wb_index = 2;
623 info->unused_entries_index = 2;
624 break;
625 case XE_METEORLAKE:
626 info->ops = &mtl_mocs_ops;
627 info->table_size = ARRAY_SIZE(mtl_mocs_desc);
628 info->table = mtl_mocs_desc;
629 info->num_mocs_regs = MTL_NUM_MOCS_ENTRIES;
630 info->uc_index = 9;
631 info->unused_entries_index = 1;
632 break;
633 case XE_DG2:
634 info->ops = &xehp_mocs_ops;
635 info->table_size = ARRAY_SIZE(dg2_mocs_desc);
636 info->table = dg2_mocs_desc;
637 info->uc_index = 1;
638 /*
639 * Last entry is RO on hardware, don't bother with what was
640 * written when checking later
641 */
642 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES - 1;
643 info->unused_entries_index = 3;
644 break;
645 case XE_DG1:
646 info->ops = &xelp_mocs_ops;
647 info->table_size = ARRAY_SIZE(dg1_mocs_desc);
648 info->table = dg1_mocs_desc;
649 info->uc_index = 1;
650 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES;
651 info->unused_entries_index = 5;
652 break;
653 case XE_TIGERLAKE:
654 case XE_ROCKETLAKE:
655 case XE_ALDERLAKE_S:
656 case XE_ALDERLAKE_P:
657 case XE_ALDERLAKE_N:
658 info->ops = &xelp_mocs_ops;
659 info->table_size = ARRAY_SIZE(gen12_mocs_desc);
660 info->table = gen12_mocs_desc;
661 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES;
662 info->uc_index = 3;
663 info->unused_entries_index = 2;
664 break;
665 default:
666 drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n");
667 return 0;
668 }
669
670 /*
671 * Index 0 is a reserved/unused table entry on most platforms, but
672 * even on those where it does represent a legitimate MOCS entry, it
673 * never represents the "most cached, least coherent" behavior we want
674 * to populate undefined table rows with. So if unused_entries_index
675 * is still 0 at this point, we'll assume that it was omitted by
676 * mistake in the switch statement above.
677 */
678 xe_assert(xe, info->unused_entries_index != 0);
679
680 xe_assert(xe, info->ops && info->ops->dump);
681 xe_assert(xe, info->table_size <= info->num_mocs_regs);
682
683 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20)
684 flags |= HAS_GLOBAL_MOCS;
685 if (GRAPHICS_VER(xe) < 20)
686 flags |= HAS_LNCF_MOCS;
687
688 return flags;
689}
690
691/*
692 * Get control_value from MOCS entry. If the table entry is not defined, the
693 * settings from unused_entries_index will be returned.
694 */
695static u32 get_entry_control(const struct xe_mocs_info *info,
696 unsigned int index)
697{
698 if (index < info->table_size && info->table[index].used)
699 return info->table[index].control_value;
700 return info->table[info->unused_entries_index].control_value;
701}
702
703static void __init_mocs_table(struct xe_gt *gt,
704 const struct xe_mocs_info *info)
705{
706 unsigned int i;
707 u32 mocs;
708
709 mocs_dbg(gt, "mocs entries: %d\n", info->num_mocs_regs);
710
711 for (i = 0; i < info->num_mocs_regs; i++) {
712 mocs = get_entry_control(info, i);
713
714 mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i,
715 XELP_GLOBAL_MOCS(i).addr, mocs);
716
717 if (regs_are_mcr(gt))
718 xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs);
719 else
720 xe_mmio_write32(>->mmio, XELP_GLOBAL_MOCS(i), mocs);
721 }
722}
723
724/*
725 * Get l3cc_value from MOCS entry taking into account when it's not used
726 * then if unused_entries_index is not zero then its value will be returned
727 * otherwise I915_MOCS_PTE's value is returned in this case.
728 */
729static u16 get_entry_l3cc(const struct xe_mocs_info *info,
730 unsigned int index)
731{
732 if (index < info->table_size && info->table[index].used)
733 return info->table[index].l3cc_value;
734 return info->table[info->unused_entries_index].l3cc_value;
735}
736
737static u32 l3cc_combine(u16 low, u16 high)
738{
739 return low | (u32)high << 16;
740}
741
742static void init_l3cc_table(struct xe_gt *gt,
743 const struct xe_mocs_info *info)
744{
745 unsigned int i;
746 u32 l3cc;
747
748 mocs_dbg(gt, "l3cc entries: %d\n", info->num_mocs_regs);
749
750 for (i = 0; i < (info->num_mocs_regs + 1) / 2; i++) {
751 l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i),
752 get_entry_l3cc(info, 2 * i + 1));
753
754 mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i,
755 XELP_LNCFCMOCS(i).addr, l3cc);
756
757 if (regs_are_mcr(gt))
758 xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc);
759 else
760 xe_mmio_write32(>->mmio, XELP_LNCFCMOCS(i), l3cc);
761 }
762}
763
764void xe_mocs_init_early(struct xe_gt *gt)
765{
766 struct xe_mocs_info table;
767
768 get_mocs_settings(gt_to_xe(gt), &table);
769 gt->mocs.uc_index = table.uc_index;
770 gt->mocs.wb_index = table.wb_index;
771}
772
773void xe_mocs_init(struct xe_gt *gt)
774{
775 struct xe_mocs_info table;
776 unsigned int flags;
777
778 if (IS_SRIOV_VF(gt_to_xe(gt)))
779 return;
780
781 /*
782 * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS"
783 * registers depending on platform.
784 *
785 * These registers should be programmed before GuC initialization
786 * since their values will affect some of the memory transactions
787 * performed by the GuC.
788 */
789 flags = get_mocs_settings(gt_to_xe(gt), &table);
790 mocs_dbg(gt, "flag:0x%x\n", flags);
791
792 if (IS_SRIOV_VF(gt_to_xe(gt)))
793 return;
794
795 if (flags & HAS_GLOBAL_MOCS)
796 __init_mocs_table(gt, &table);
797 if (flags & HAS_LNCF_MOCS)
798 init_l3cc_table(gt, &table);
799}
800
801/**
802 * xe_mocs_dump() - Dump MOCS table.
803 * @gt: the &xe_gt with MOCS table
804 * @p: the &drm_printer to dump info to
805 *
806 * Return: 0 on success or a negative error code on failure.
807 */
808int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p)
809{
810 struct xe_device *xe = gt_to_xe(gt);
811 enum xe_force_wake_domains domain;
812 struct xe_mocs_info table;
813 unsigned int flags;
814
815 flags = get_mocs_settings(xe, &table);
816
817 domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT;
818
819 guard(xe_pm_runtime_noresume)(xe);
820 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain);
821 if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain))
822 return -ETIMEDOUT;
823
824 table.ops->dump(&table, flags, gt, p);
825
826 return 0;
827}
828
829#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
830#include "tests/xe_mocs.c"
831#endif