Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_irq.h"
7
8#include <linux/sched/clock.h>
9
10#include <drm/drm_managed.h>
11
12#include "display/xe_display.h"
13#include "regs/xe_irq_regs.h"
14#include "xe_device.h"
15#include "xe_drv.h"
16#include "xe_gsc_proxy.h"
17#include "xe_gt.h"
18#include "xe_guc.h"
19#include "xe_hw_engine.h"
20#include "xe_hw_error.h"
21#include "xe_i2c.h"
22#include "xe_memirq.h"
23#include "xe_mert.h"
24#include "xe_mmio.h"
25#include "xe_pxp.h"
26#include "xe_sriov.h"
27#include "xe_tile.h"
28
29/*
30 * Interrupt registers for a unit are always consecutive and ordered
31 * ISR, IMR, IIR, IER.
32 */
33#define IMR(offset) XE_REG(offset + 0x4)
34#define IIR(offset) XE_REG(offset + 0x8)
35#define IER(offset) XE_REG(offset + 0xc)
36
37static int xe_irq_msix_init(struct xe_device *xe);
38static void xe_irq_msix_free(struct xe_device *xe);
39static int xe_irq_msix_request_irqs(struct xe_device *xe);
40static void xe_irq_msix_synchronize_irq(struct xe_device *xe);
41
42static void assert_iir_is_zero(struct xe_mmio *mmio, struct xe_reg reg)
43{
44 u32 val = xe_mmio_read32(mmio, reg);
45
46 if (val == 0)
47 return;
48
49 drm_WARN(&mmio->tile->xe->drm, 1,
50 "Interrupt register 0x%x is not zero: 0x%08x\n",
51 reg.addr, val);
52 xe_mmio_write32(mmio, reg, 0xffffffff);
53 xe_mmio_read32(mmio, reg);
54 xe_mmio_write32(mmio, reg, 0xffffffff);
55 xe_mmio_read32(mmio, reg);
56}
57
58/*
59 * Unmask and enable the specified interrupts. Does not check current state,
60 * so any bits not specified here will become masked and disabled.
61 */
62static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits)
63{
64 struct xe_mmio *mmio = &tile->mmio;
65
66 /*
67 * If we're just enabling an interrupt now, it shouldn't already
68 * be raised in the IIR.
69 */
70 assert_iir_is_zero(mmio, IIR(irqregs));
71
72 xe_mmio_write32(mmio, IER(irqregs), bits);
73 xe_mmio_write32(mmio, IMR(irqregs), ~bits);
74
75 /* Posting read */
76 xe_mmio_read32(mmio, IMR(irqregs));
77}
78
79/* Mask and disable all interrupts. */
80static void mask_and_disable(struct xe_tile *tile, u32 irqregs)
81{
82 struct xe_mmio *mmio = &tile->mmio;
83
84 xe_mmio_write32(mmio, IMR(irqregs), ~0);
85 /* Posting read */
86 xe_mmio_read32(mmio, IMR(irqregs));
87
88 xe_mmio_write32(mmio, IER(irqregs), 0);
89
90 /* IIR can theoretically queue up two events. Be paranoid. */
91 xe_mmio_write32(mmio, IIR(irqregs), ~0);
92 xe_mmio_read32(mmio, IIR(irqregs));
93 xe_mmio_write32(mmio, IIR(irqregs), ~0);
94 xe_mmio_read32(mmio, IIR(irqregs));
95}
96
97static u32 xelp_intr_disable(struct xe_device *xe)
98{
99 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
100
101 xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0);
102
103 /*
104 * Now with master disabled, get a sample of level indications
105 * for this interrupt. Indications will be cleared on related acks.
106 * New indications can and will light up during processing,
107 * and will generate new interrupt after enabling master.
108 */
109 return xe_mmio_read32(mmio, GFX_MSTR_IRQ);
110}
111
112static u32
113gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl)
114{
115 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
116 u32 iir;
117
118 if (!(master_ctl & GU_MISC_IRQ))
119 return 0;
120
121 iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET));
122 if (likely(iir))
123 xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir);
124
125 return iir;
126}
127
128static inline void xelp_intr_enable(struct xe_device *xe, bool stall)
129{
130 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
131
132 xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ);
133 if (stall)
134 xe_mmio_read32(mmio, GFX_MSTR_IRQ);
135}
136
137/* Enable/unmask the HWE interrupts for a specific GT's engines. */
138void xe_irq_enable_hwe(struct xe_gt *gt)
139{
140 struct xe_device *xe = gt_to_xe(gt);
141 struct xe_mmio *mmio = >->mmio;
142 u32 common_mask, val, gsc_mask = 0, heci_mask = 0,
143 rcs_mask = 0, bcs_mask = 0, vcs_mask = 0, vecs_mask = 0,
144 ccs_mask = 0;
145
146 if (xe_device_uses_memirq(xe))
147 return;
148
149 if (xe_device_uc_enabled(xe)) {
150 common_mask = GT_MI_USER_INTERRUPT |
151 GT_FLUSH_COMPLETE_INTERRUPT;
152
153 /* Enable Compute Walker Interrupt for non-MSIX platforms */
154 if (GRAPHICS_VERx100(xe) >= 3511 && !xe_device_has_msix(xe)) {
155 rcs_mask |= GT_COMPUTE_WALKER_INTERRUPT;
156 ccs_mask |= GT_COMPUTE_WALKER_INTERRUPT;
157 }
158 } else {
159 common_mask = GT_MI_USER_INTERRUPT |
160 GT_CS_MASTER_ERROR_INTERRUPT |
161 GT_CONTEXT_SWITCH_INTERRUPT |
162 GT_WAIT_SEMAPHORE_INTERRUPT;
163 }
164
165 rcs_mask |= common_mask;
166 bcs_mask |= common_mask;
167 vcs_mask |= common_mask;
168 vecs_mask |= common_mask;
169 ccs_mask |= common_mask;
170
171 if (xe_gt_is_main_type(gt)) {
172 /*
173 * For enabling the interrupts, the information about fused off
174 * engines doesn't matter much, but this also allows to check if
175 * the engine is available architecturally in the platform
176 */
177 u32 ccs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE);
178 u32 bcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY);
179
180 /* Enable interrupts for each engine class */
181 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE,
182 REG_FIELD_PREP(ENGINE1_MASK, rcs_mask) |
183 REG_FIELD_PREP(ENGINE0_MASK, bcs_mask));
184 if (ccs_fuse_mask)
185 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE,
186 REG_FIELD_PREP(ENGINE1_MASK, ccs_mask));
187
188 /* Unmask interrupts for each engine instance */
189 val = ~REG_FIELD_PREP(ENGINE1_MASK, rcs_mask);
190 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, val);
191 val = ~REG_FIELD_PREP(ENGINE1_MASK, bcs_mask);
192 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, val);
193
194 val = ~(REG_FIELD_PREP(ENGINE1_MASK, bcs_mask) |
195 REG_FIELD_PREP(ENGINE0_MASK, bcs_mask));
196 if (bcs_fuse_mask & (BIT(1)|BIT(2)))
197 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, val);
198 if (bcs_fuse_mask & (BIT(3)|BIT(4)))
199 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, val);
200 if (bcs_fuse_mask & (BIT(5)|BIT(6)))
201 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, val);
202 if (bcs_fuse_mask & (BIT(7)|BIT(8)))
203 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, val);
204
205 val = ~(REG_FIELD_PREP(ENGINE1_MASK, ccs_mask) |
206 REG_FIELD_PREP(ENGINE0_MASK, ccs_mask));
207 if (ccs_fuse_mask & (BIT(0)|BIT(1)))
208 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, val);
209 if (ccs_fuse_mask & (BIT(2)|BIT(3)))
210 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, val);
211 }
212
213 if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) {
214 u32 vcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
215 u32 vecs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
216 u32 other_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER);
217
218 /* Enable interrupts for each engine class */
219 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE,
220 REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) |
221 REG_FIELD_PREP(ENGINE0_MASK, vecs_mask));
222
223 /* Unmask interrupts for each engine instance */
224 val = ~(REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) |
225 REG_FIELD_PREP(ENGINE0_MASK, vcs_mask));
226 if (vcs_fuse_mask & (BIT(0) | BIT(1)))
227 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, val);
228 if (vcs_fuse_mask & (BIT(2) | BIT(3)))
229 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, val);
230 if (vcs_fuse_mask & (BIT(4) | BIT(5)))
231 xe_mmio_write32(mmio, VCS4_VCS5_INTR_MASK, val);
232 if (vcs_fuse_mask & (BIT(6) | BIT(7)))
233 xe_mmio_write32(mmio, VCS6_VCS7_INTR_MASK, val);
234
235 val = ~(REG_FIELD_PREP(ENGINE1_MASK, vecs_mask) |
236 REG_FIELD_PREP(ENGINE0_MASK, vecs_mask));
237 if (vecs_fuse_mask & (BIT(0) | BIT(1)))
238 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, val);
239 if (vecs_fuse_mask & (BIT(2) | BIT(3)))
240 xe_mmio_write32(mmio, VECS2_VECS3_INTR_MASK, val);
241
242 /*
243 * the heci2 interrupt is enabled via the same register as the
244 * GSCCS interrupts, but it has its own mask register.
245 */
246 if (other_fuse_mask) {
247 gsc_mask = common_mask | GSC_ER_COMPLETE;
248 heci_mask = GSC_IRQ_INTF(1);
249 } else if (xe->info.has_heci_gscfi) {
250 gsc_mask = GSC_IRQ_INTF(1);
251 }
252
253 if (gsc_mask) {
254 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask);
255 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~gsc_mask);
256 }
257 if (heci_mask)
258 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16));
259
260 if (xe_pxp_is_supported(xe)) {
261 u32 kcr_mask = KCR_PXP_STATE_TERMINATED_INTERRUPT |
262 KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT |
263 KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT;
264
265 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, kcr_mask << 16);
266 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~(kcr_mask << 16));
267 }
268 }
269}
270
271static u32
272gt_engine_identity(struct xe_device *xe,
273 struct xe_mmio *mmio,
274 const unsigned int bank,
275 const unsigned int bit)
276{
277 u32 timeout_ts;
278 u32 ident;
279
280 lockdep_assert_held(&xe->irq.lock);
281
282 xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit));
283
284 /*
285 * NB: Specs do not specify how long to spin wait,
286 * so we do ~100us as an educated guess.
287 */
288 timeout_ts = (local_clock() >> 10) + 100;
289 do {
290 ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank));
291 } while (!(ident & INTR_DATA_VALID) &&
292 !time_after32(local_clock() >> 10, timeout_ts));
293
294 if (unlikely(!(ident & INTR_DATA_VALID))) {
295 drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
296 bank, bit, ident);
297 return 0;
298 }
299
300 xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident);
301
302 return ident;
303}
304
305#define OTHER_MEDIA_GUC_INSTANCE 16
306
307static void
308gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir)
309{
310 if (instance == OTHER_GUC_INSTANCE && xe_gt_is_main_type(gt))
311 return xe_guc_irq_handler(>->uc.guc, iir);
312 if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt))
313 return xe_guc_irq_handler(>->uc.guc, iir);
314 if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt))
315 return xe_gsc_proxy_irq_handler(>->uc.gsc, iir);
316
317 if (instance != OTHER_GUC_INSTANCE &&
318 instance != OTHER_MEDIA_GUC_INSTANCE) {
319 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
320 instance, iir);
321 }
322}
323
324static struct xe_gt *pick_engine_gt(struct xe_tile *tile,
325 enum xe_engine_class class,
326 unsigned int instance)
327{
328 struct xe_device *xe = tile_to_xe(tile);
329
330 if (MEDIA_VER(xe) < 13)
331 return tile->primary_gt;
332
333 switch (class) {
334 case XE_ENGINE_CLASS_VIDEO_DECODE:
335 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
336 return tile->media_gt;
337 case XE_ENGINE_CLASS_OTHER:
338 switch (instance) {
339 case OTHER_MEDIA_GUC_INSTANCE:
340 case OTHER_GSC_INSTANCE:
341 case OTHER_GSC_HECI2_INSTANCE:
342 return tile->media_gt;
343 default:
344 break;
345 }
346 fallthrough;
347 default:
348 return tile->primary_gt;
349 }
350}
351
352static void gt_irq_handler(struct xe_tile *tile,
353 u32 master_ctl, unsigned long *intr_dw,
354 u32 *identity)
355{
356 struct xe_device *xe = tile_to_xe(tile);
357 struct xe_mmio *mmio = &tile->mmio;
358 unsigned int bank, bit;
359 u16 instance, intr_vec;
360 enum xe_engine_class class;
361 struct xe_hw_engine *hwe;
362
363 spin_lock(&xe->irq.lock);
364
365 for (bank = 0; bank < 2; bank++) {
366 if (!(master_ctl & GT_DW_IRQ(bank)))
367 continue;
368
369 intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank));
370 for_each_set_bit(bit, intr_dw + bank, 32)
371 identity[bit] = gt_engine_identity(xe, mmio, bank, bit);
372 xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]);
373
374 for_each_set_bit(bit, intr_dw + bank, 32) {
375 struct xe_gt *engine_gt;
376
377 class = INTR_ENGINE_CLASS(identity[bit]);
378 instance = INTR_ENGINE_INSTANCE(identity[bit]);
379 intr_vec = INTR_ENGINE_INTR(identity[bit]);
380
381 engine_gt = pick_engine_gt(tile, class, instance);
382
383 hwe = xe_gt_hw_engine(engine_gt, class, instance, false);
384 if (hwe) {
385 xe_hw_engine_handle_irq(hwe, intr_vec);
386 continue;
387 }
388
389 if (class == XE_ENGINE_CLASS_OTHER) {
390 /*
391 * HECI GSCFI interrupts come from outside of GT.
392 * KCR irqs come from inside GT but are handled
393 * by the global PXP subsystem.
394 */
395 if (xe->info.has_heci_gscfi && instance == OTHER_GSC_INSTANCE)
396 xe_heci_gsc_irq_handler(xe, intr_vec);
397 else if (instance == OTHER_KCR_INSTANCE)
398 xe_pxp_irq_handler(xe, intr_vec);
399 else
400 gt_other_irq_handler(engine_gt, instance, intr_vec);
401 }
402 }
403 }
404
405 spin_unlock(&xe->irq.lock);
406}
407
408/*
409 * Top-level interrupt handler for Xe_LP platforms (which did not have
410 * a "master tile" interrupt register.
411 */
412static irqreturn_t xelp_irq_handler(int irq, void *arg)
413{
414 struct xe_device *xe = arg;
415 struct xe_tile *tile = xe_device_get_root_tile(xe);
416 u32 master_ctl, gu_misc_iir;
417 unsigned long intr_dw[2];
418 u32 identity[32];
419
420 if (!atomic_read(&xe->irq.enabled))
421 return IRQ_NONE;
422
423 master_ctl = xelp_intr_disable(xe);
424 if (!master_ctl) {
425 xelp_intr_enable(xe, false);
426 return IRQ_NONE;
427 }
428
429 gt_irq_handler(tile, master_ctl, intr_dw, identity);
430
431 xe_display_irq_handler(xe, master_ctl);
432
433 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
434
435 xelp_intr_enable(xe, false);
436
437 xe_display_irq_enable(xe, gu_misc_iir);
438
439 return IRQ_HANDLED;
440}
441
442static u32 dg1_intr_disable(struct xe_device *xe)
443{
444 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
445 u32 val;
446
447 /* First disable interrupts */
448 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0);
449
450 /* Get the indication levels and ack the master unit */
451 val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
452 if (unlikely(!val))
453 return 0;
454
455 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val);
456
457 return val;
458}
459
460static void dg1_intr_enable(struct xe_device *xe, bool stall)
461{
462 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
463
464 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
465 if (stall)
466 xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
467}
468
469/*
470 * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have
471 * a "master tile" interrupt register which must be consulted before the
472 * "graphics master" interrupt register.
473 */
474static irqreturn_t dg1_irq_handler(int irq, void *arg)
475{
476 struct xe_device *xe = arg;
477 struct xe_tile *tile;
478 u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0;
479 unsigned long intr_dw[2];
480 u32 identity[32];
481 u8 id;
482
483 /* TODO: This really shouldn't be copied+pasted */
484
485 if (!atomic_read(&xe->irq.enabled))
486 return IRQ_NONE;
487
488 master_tile_ctl = dg1_intr_disable(xe);
489 if (!master_tile_ctl) {
490 dg1_intr_enable(xe, false);
491 return IRQ_NONE;
492 }
493
494 for_each_tile(tile, xe, id) {
495 struct xe_mmio *mmio = &tile->mmio;
496
497 if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0)
498 continue;
499
500 master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ);
501
502 /*
503 * We might be in irq handler just when PCIe DPC is initiated
504 * and all MMIO reads will be returned with all 1's. Ignore this
505 * irq as device is inaccessible.
506 */
507 if (master_ctl == REG_GENMASK(31, 0)) {
508 drm_dbg(&tile_to_xe(tile)->drm,
509 "Ignore this IRQ as device might be in DPC containment.\n");
510 return IRQ_HANDLED;
511 }
512
513 xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl);
514
515 gt_irq_handler(tile, master_ctl, intr_dw, identity);
516 xe_hw_error_irq_handler(tile, master_ctl);
517
518 /*
519 * Display interrupts (including display backlight operations
520 * that get reported as Gunit GSE) would only be hooked up to
521 * the primary tile.
522 */
523 if (id == 0) {
524 if (xe->info.has_heci_cscfi)
525 xe_heci_csc_irq_handler(xe, master_ctl);
526 xe_display_irq_handler(xe, master_ctl);
527 xe_i2c_irq_handler(xe, master_ctl);
528 xe_mert_irq_handler(xe, master_ctl);
529 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
530 }
531 }
532
533 dg1_intr_enable(xe, false);
534 xe_display_irq_enable(xe, gu_misc_iir);
535
536 return IRQ_HANDLED;
537}
538
539static void gt_irq_reset(struct xe_tile *tile)
540{
541 struct xe_mmio *mmio = &tile->mmio;
542 u32 ccs_mask = ~0;
543 u32 bcs_mask = ~0;
544
545 if (tile->primary_gt) {
546 ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
547 XE_ENGINE_CLASS_COMPUTE);
548 bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
549 XE_ENGINE_CLASS_COPY);
550 }
551
552 /* Disable RCS, BCS, VCS and VECS class engines. */
553 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0);
554 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0);
555 if (ccs_mask)
556 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0);
557
558 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
559 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0);
560 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0);
561 if (bcs_mask & (BIT(1)|BIT(2)))
562 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
563 if (bcs_mask & (BIT(3)|BIT(4)))
564 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
565 if (bcs_mask & (BIT(5)|BIT(6)))
566 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
567 if (bcs_mask & (BIT(7)|BIT(8)))
568 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
569 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0);
570 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0);
571 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0);
572 if (ccs_mask & (BIT(0)|BIT(1)))
573 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0);
574 if (ccs_mask & (BIT(2)|BIT(3)))
575 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0);
576
577 if ((tile->media_gt &&
578 xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) ||
579 tile_to_xe(tile)->info.has_heci_gscfi) {
580 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0);
581 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0);
582 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0);
583 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, 0);
584 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~0);
585 }
586
587 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0);
588 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0);
589 xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0);
590 xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0);
591}
592
593static void xelp_irq_reset(struct xe_tile *tile)
594{
595 xelp_intr_disable(tile_to_xe(tile));
596
597 gt_irq_reset(tile);
598
599 if (IS_SRIOV_VF(tile_to_xe(tile)))
600 return;
601
602 mask_and_disable(tile, PCU_IRQ_OFFSET);
603}
604
605static void dg1_irq_reset(struct xe_tile *tile)
606{
607 if (xe_tile_is_root(tile))
608 dg1_intr_disable(tile_to_xe(tile));
609
610 gt_irq_reset(tile);
611
612 if (IS_SRIOV_VF(tile_to_xe(tile)))
613 return;
614
615 mask_and_disable(tile, PCU_IRQ_OFFSET);
616}
617
618static void dg1_irq_reset_mstr(struct xe_tile *tile)
619{
620 struct xe_mmio *mmio = &tile->mmio;
621
622 xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0);
623}
624
625static void vf_irq_reset(struct xe_device *xe)
626{
627 struct xe_tile *tile;
628 unsigned int id;
629
630 xe_assert(xe, IS_SRIOV_VF(xe));
631
632 if (GRAPHICS_VERx100(xe) < 1210)
633 xelp_intr_disable(xe);
634 else
635 xe_assert(xe, xe_device_has_memirq(xe));
636
637 for_each_tile(tile, xe, id) {
638 if (xe_device_has_memirq(xe))
639 xe_memirq_reset(&tile->memirq);
640 else
641 gt_irq_reset(tile);
642 }
643}
644
645static void xe_irq_reset(struct xe_device *xe)
646{
647 struct xe_tile *tile;
648 u8 id;
649
650 if (IS_SRIOV_VF(xe))
651 return vf_irq_reset(xe);
652
653 if (xe_device_uses_memirq(xe)) {
654 for_each_tile(tile, xe, id)
655 xe_memirq_reset(&tile->memirq);
656 }
657
658 for_each_tile(tile, xe, id) {
659 if (GRAPHICS_VERx100(xe) >= 1210)
660 dg1_irq_reset(tile);
661 else
662 xelp_irq_reset(tile);
663 }
664
665 tile = xe_device_get_root_tile(xe);
666 mask_and_disable(tile, GU_MISC_IRQ_OFFSET);
667 xe_display_irq_reset(xe);
668 xe_i2c_irq_reset(xe);
669
670 /*
671 * The tile's top-level status register should be the last one
672 * to be reset to avoid possible bit re-latching from lower
673 * level interrupts.
674 */
675 if (GRAPHICS_VERx100(xe) >= 1210) {
676 for_each_tile(tile, xe, id)
677 dg1_irq_reset_mstr(tile);
678 }
679}
680
681static void vf_irq_postinstall(struct xe_device *xe)
682{
683 struct xe_tile *tile;
684 unsigned int id;
685
686 for_each_tile(tile, xe, id)
687 if (xe_device_has_memirq(xe))
688 xe_memirq_postinstall(&tile->memirq);
689
690 if (GRAPHICS_VERx100(xe) < 1210)
691 xelp_intr_enable(xe, true);
692 else
693 xe_assert(xe, xe_device_has_memirq(xe));
694}
695
696static void xe_irq_postinstall(struct xe_device *xe)
697{
698 if (IS_SRIOV_VF(xe))
699 return vf_irq_postinstall(xe);
700
701 if (xe_device_uses_memirq(xe)) {
702 struct xe_tile *tile;
703 unsigned int id;
704
705 for_each_tile(tile, xe, id)
706 xe_memirq_postinstall(&tile->memirq);
707 }
708
709 xe_display_irq_postinstall(xe);
710 xe_i2c_irq_postinstall(xe);
711
712 /*
713 * ASLE backlight operations are reported via GUnit GSE interrupts
714 * on the root tile.
715 */
716 unmask_and_enable(xe_device_get_root_tile(xe),
717 GU_MISC_IRQ_OFFSET, GU_MISC_GSE);
718
719 /* Enable top-level interrupts */
720 if (GRAPHICS_VERx100(xe) >= 1210)
721 dg1_intr_enable(xe, true);
722 else
723 xelp_intr_enable(xe, true);
724}
725
726static irqreturn_t vf_mem_irq_handler(int irq, void *arg)
727{
728 struct xe_device *xe = arg;
729 struct xe_tile *tile;
730 unsigned int id;
731
732 if (!atomic_read(&xe->irq.enabled))
733 return IRQ_NONE;
734
735 for_each_tile(tile, xe, id)
736 xe_memirq_handler(&tile->memirq);
737
738 return IRQ_HANDLED;
739}
740
741static irq_handler_t xe_irq_handler(struct xe_device *xe)
742{
743 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe))
744 return vf_mem_irq_handler;
745
746 if (GRAPHICS_VERx100(xe) >= 1210)
747 return dg1_irq_handler;
748 else
749 return xelp_irq_handler;
750}
751
752static int xe_irq_msi_request_irqs(struct xe_device *xe)
753{
754 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
755 irq_handler_t irq_handler;
756 int irq, err;
757
758 irq_handler = xe_irq_handler(xe);
759 if (!irq_handler) {
760 drm_err(&xe->drm, "No supported interrupt handler");
761 return -EINVAL;
762 }
763
764 irq = pci_irq_vector(pdev, 0);
765 err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe);
766 if (err < 0) {
767 drm_err(&xe->drm, "Failed to request MSI IRQ %d\n", err);
768 return err;
769 }
770
771 return 0;
772}
773
774static void xe_irq_msi_free(struct xe_device *xe)
775{
776 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
777 int irq;
778
779 irq = pci_irq_vector(pdev, 0);
780 free_irq(irq, xe);
781}
782
783static void irq_uninstall(void *arg)
784{
785 struct xe_device *xe = arg;
786
787 if (!atomic_xchg(&xe->irq.enabled, 0))
788 return;
789
790 xe_irq_reset(xe);
791
792 if (xe_device_has_msix(xe))
793 xe_irq_msix_free(xe);
794 else
795 xe_irq_msi_free(xe);
796}
797
798int xe_irq_init(struct xe_device *xe)
799{
800 spin_lock_init(&xe->irq.lock);
801
802 return xe_irq_msix_init(xe);
803}
804
805int xe_irq_install(struct xe_device *xe)
806{
807 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
808 unsigned int irq_flags = PCI_IRQ_MSI;
809 int nvec = 1;
810 int err;
811
812 xe_hw_error_init(xe);
813
814 xe_irq_reset(xe);
815
816 if (xe_device_has_msix(xe)) {
817 nvec = xe->irq.msix.nvec;
818 irq_flags = PCI_IRQ_MSIX;
819 }
820
821 err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags);
822 if (err < 0) {
823 drm_err(&xe->drm, "Failed to allocate IRQ vectors: %d\n", err);
824 return err;
825 }
826
827 err = xe_device_has_msix(xe) ? xe_irq_msix_request_irqs(xe) :
828 xe_irq_msi_request_irqs(xe);
829 if (err)
830 return err;
831
832 atomic_set(&xe->irq.enabled, 1);
833
834 xe_irq_postinstall(xe);
835
836 return devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe);
837}
838
839static void xe_irq_msi_synchronize_irq(struct xe_device *xe)
840{
841 synchronize_irq(to_pci_dev(xe->drm.dev)->irq);
842}
843
844void xe_irq_suspend(struct xe_device *xe)
845{
846 atomic_set(&xe->irq.enabled, 0); /* no new irqs */
847
848 /* flush irqs */
849 if (xe_device_has_msix(xe))
850 xe_irq_msix_synchronize_irq(xe);
851 else
852 xe_irq_msi_synchronize_irq(xe);
853 xe_irq_reset(xe); /* turn irqs off */
854}
855
856void xe_irq_resume(struct xe_device *xe)
857{
858 struct xe_gt *gt;
859 int id;
860
861 /*
862 * lock not needed:
863 * 1. no irq will arrive before the postinstall
864 * 2. display is not yet resumed
865 */
866 atomic_set(&xe->irq.enabled, 1);
867 xe_irq_reset(xe);
868 xe_irq_postinstall(xe); /* turn irqs on */
869
870 for_each_gt(gt, xe, id)
871 xe_irq_enable_hwe(gt);
872}
873
874/* MSI-X related definitions and functions below. */
875
876enum xe_irq_msix_static {
877 GUC2HOST_MSIX = 0,
878 DEFAULT_MSIX = XE_IRQ_DEFAULT_MSIX,
879 /* Must be last */
880 NUM_OF_STATIC_MSIX,
881};
882
883static int xe_irq_msix_init(struct xe_device *xe)
884{
885 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
886 int nvec = pci_msix_vec_count(pdev);
887
888 if (nvec == -EINVAL)
889 return 0; /* MSI */
890
891 if (nvec < 0) {
892 drm_err(&xe->drm, "Failed getting MSI-X vectors count: %d\n", nvec);
893 return nvec;
894 }
895
896 xe->irq.msix.nvec = nvec;
897 xa_init_flags(&xe->irq.msix.indexes, XA_FLAGS_ALLOC);
898 return 0;
899}
900
901static irqreturn_t xe_irq_msix_default_hwe_handler(int irq, void *arg)
902{
903 unsigned int tile_id, gt_id;
904 struct xe_device *xe = arg;
905 struct xe_memirq *memirq;
906 struct xe_hw_engine *hwe;
907 enum xe_hw_engine_id id;
908 struct xe_tile *tile;
909 struct xe_gt *gt;
910
911 if (!atomic_read(&xe->irq.enabled))
912 return IRQ_NONE;
913
914 for_each_tile(tile, xe, tile_id) {
915 memirq = &tile->memirq;
916 if (!memirq->bo)
917 continue;
918
919 for_each_gt(gt, xe, gt_id) {
920 if (gt->tile != tile)
921 continue;
922
923 for_each_hw_engine(hwe, gt, id)
924 xe_memirq_hwe_handler(memirq, hwe);
925 }
926 }
927
928 return IRQ_HANDLED;
929}
930
931static int xe_irq_msix_alloc_vector(struct xe_device *xe, void *irq_buf,
932 bool dynamic_msix, u16 *msix)
933{
934 struct xa_limit limit;
935 int ret;
936 u32 id;
937
938 limit = (dynamic_msix) ? XA_LIMIT(NUM_OF_STATIC_MSIX, xe->irq.msix.nvec - 1) :
939 XA_LIMIT(*msix, *msix);
940 ret = xa_alloc(&xe->irq.msix.indexes, &id, irq_buf, limit, GFP_KERNEL);
941 if (ret)
942 return ret;
943
944 if (dynamic_msix)
945 *msix = id;
946
947 return 0;
948}
949
950static void xe_irq_msix_release_vector(struct xe_device *xe, u16 msix)
951{
952 xa_erase(&xe->irq.msix.indexes, msix);
953}
954
955static int xe_irq_msix_request_irq_internal(struct xe_device *xe, irq_handler_t handler,
956 void *irq_buf, const char *name, u16 msix)
957{
958 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
959 int ret, irq;
960
961 irq = pci_irq_vector(pdev, msix);
962 if (irq < 0)
963 return irq;
964
965 ret = request_irq(irq, handler, IRQF_SHARED, name, irq_buf);
966 if (ret < 0)
967 return ret;
968
969 return 0;
970}
971
972int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf,
973 const char *name, bool dynamic_msix, u16 *msix)
974{
975 int ret;
976
977 ret = xe_irq_msix_alloc_vector(xe, irq_buf, dynamic_msix, msix);
978 if (ret)
979 return ret;
980
981 ret = xe_irq_msix_request_irq_internal(xe, handler, irq_buf, name, *msix);
982 if (ret) {
983 drm_err(&xe->drm, "Failed to request IRQ for MSI-X %u\n", *msix);
984 xe_irq_msix_release_vector(xe, *msix);
985 return ret;
986 }
987
988 return 0;
989}
990
991void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix)
992{
993 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
994 int irq;
995 void *irq_buf;
996
997 irq_buf = xa_load(&xe->irq.msix.indexes, msix);
998 if (!irq_buf)
999 return;
1000
1001 irq = pci_irq_vector(pdev, msix);
1002 if (irq < 0) {
1003 drm_err(&xe->drm, "MSI-X %u can't be released, there is no matching IRQ\n", msix);
1004 return;
1005 }
1006
1007 free_irq(irq, irq_buf);
1008 xe_irq_msix_release_vector(xe, msix);
1009}
1010
1011int xe_irq_msix_request_irqs(struct xe_device *xe)
1012{
1013 int err;
1014 u16 msix;
1015
1016 msix = GUC2HOST_MSIX;
1017 err = xe_irq_msix_request_irq(xe, xe_irq_handler(xe), xe,
1018 DRIVER_NAME "-guc2host", false, &msix);
1019 if (err)
1020 return err;
1021
1022 msix = DEFAULT_MSIX;
1023 err = xe_irq_msix_request_irq(xe, xe_irq_msix_default_hwe_handler, xe,
1024 DRIVER_NAME "-default-msix", false, &msix);
1025 if (err) {
1026 xe_irq_msix_free_irq(xe, GUC2HOST_MSIX);
1027 return err;
1028 }
1029
1030 return 0;
1031}
1032
1033void xe_irq_msix_free(struct xe_device *xe)
1034{
1035 unsigned long msix;
1036 u32 *dummy;
1037
1038 xa_for_each(&xe->irq.msix.indexes, msix, dummy)
1039 xe_irq_msix_free_irq(xe, msix);
1040 xa_destroy(&xe->irq.msix.indexes);
1041}
1042
1043void xe_irq_msix_synchronize_irq(struct xe_device *xe)
1044{
1045 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
1046 unsigned long msix;
1047 u32 *dummy;
1048
1049 xa_for_each(&xe->irq.msix.indexes, msix, dummy)
1050 synchronize_irq(pci_irq_vector(pdev, msix));
1051}