Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_irq.h"
7
8#include <linux/sched/clock.h>
9
10#include <drm/drm_managed.h>
11
12#include "display/xe_display.h"
13#include "regs/xe_gt_regs.h"
14#include "regs/xe_regs.h"
15#include "xe_device.h"
16#include "xe_drv.h"
17#include "xe_gsc_proxy.h"
18#include "xe_gt.h"
19#include "xe_guc.h"
20#include "xe_hw_engine.h"
21#include "xe_memirq.h"
22#include "xe_mmio.h"
23#include "xe_sriov.h"
24
25/*
26 * Interrupt registers for a unit are always consecutive and ordered
27 * ISR, IMR, IIR, IER.
28 */
29#define IMR(offset) XE_REG(offset + 0x4)
30#define IIR(offset) XE_REG(offset + 0x8)
31#define IER(offset) XE_REG(offset + 0xc)
32
33static void assert_iir_is_zero(struct xe_gt *mmio, struct xe_reg reg)
34{
35 u32 val = xe_mmio_read32(mmio, reg);
36
37 if (val == 0)
38 return;
39
40 drm_WARN(>_to_xe(mmio)->drm, 1,
41 "Interrupt register 0x%x is not zero: 0x%08x\n",
42 reg.addr, val);
43 xe_mmio_write32(mmio, reg, 0xffffffff);
44 xe_mmio_read32(mmio, reg);
45 xe_mmio_write32(mmio, reg, 0xffffffff);
46 xe_mmio_read32(mmio, reg);
47}
48
49/*
50 * Unmask and enable the specified interrupts. Does not check current state,
51 * so any bits not specified here will become masked and disabled.
52 */
53static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits)
54{
55 struct xe_gt *mmio = tile->primary_gt;
56
57 /*
58 * If we're just enabling an interrupt now, it shouldn't already
59 * be raised in the IIR.
60 */
61 assert_iir_is_zero(mmio, IIR(irqregs));
62
63 xe_mmio_write32(mmio, IER(irqregs), bits);
64 xe_mmio_write32(mmio, IMR(irqregs), ~bits);
65
66 /* Posting read */
67 xe_mmio_read32(mmio, IMR(irqregs));
68}
69
70/* Mask and disable all interrupts. */
71static void mask_and_disable(struct xe_tile *tile, u32 irqregs)
72{
73 struct xe_gt *mmio = tile->primary_gt;
74
75 xe_mmio_write32(mmio, IMR(irqregs), ~0);
76 /* Posting read */
77 xe_mmio_read32(mmio, IMR(irqregs));
78
79 xe_mmio_write32(mmio, IER(irqregs), 0);
80
81 /* IIR can theoretically queue up two events. Be paranoid. */
82 xe_mmio_write32(mmio, IIR(irqregs), ~0);
83 xe_mmio_read32(mmio, IIR(irqregs));
84 xe_mmio_write32(mmio, IIR(irqregs), ~0);
85 xe_mmio_read32(mmio, IIR(irqregs));
86}
87
88static u32 xelp_intr_disable(struct xe_device *xe)
89{
90 struct xe_gt *mmio = xe_root_mmio_gt(xe);
91
92 xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0);
93
94 /*
95 * Now with master disabled, get a sample of level indications
96 * for this interrupt. Indications will be cleared on related acks.
97 * New indications can and will light up during processing,
98 * and will generate new interrupt after enabling master.
99 */
100 return xe_mmio_read32(mmio, GFX_MSTR_IRQ);
101}
102
103static u32
104gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl)
105{
106 struct xe_gt *mmio = xe_root_mmio_gt(xe);
107 u32 iir;
108
109 if (!(master_ctl & GU_MISC_IRQ))
110 return 0;
111
112 iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET));
113 if (likely(iir))
114 xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir);
115
116 return iir;
117}
118
119static inline void xelp_intr_enable(struct xe_device *xe, bool stall)
120{
121 struct xe_gt *mmio = xe_root_mmio_gt(xe);
122
123 xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ);
124 if (stall)
125 xe_mmio_read32(mmio, GFX_MSTR_IRQ);
126}
127
128/* Enable/unmask the HWE interrupts for a specific GT's engines. */
129void xe_irq_enable_hwe(struct xe_gt *gt)
130{
131 struct xe_device *xe = gt_to_xe(gt);
132 u32 ccs_mask, bcs_mask;
133 u32 irqs, dmask, smask;
134 u32 gsc_mask = 0;
135 u32 heci_mask = 0;
136
137 if (xe_device_uc_enabled(xe)) {
138 irqs = GT_RENDER_USER_INTERRUPT |
139 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
140 } else {
141 irqs = GT_RENDER_USER_INTERRUPT |
142 GT_CS_MASTER_ERROR_INTERRUPT |
143 GT_CONTEXT_SWITCH_INTERRUPT |
144 GT_WAIT_SEMAPHORE_INTERRUPT;
145 }
146
147 ccs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE);
148 bcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY);
149
150 dmask = irqs << 16 | irqs;
151 smask = irqs << 16;
152
153 if (!xe_gt_is_media_type(gt)) {
154 /* Enable interrupts for each engine class */
155 xe_mmio_write32(gt, RENDER_COPY_INTR_ENABLE, dmask);
156 if (ccs_mask)
157 xe_mmio_write32(gt, CCS_RSVD_INTR_ENABLE, smask);
158
159 /* Unmask interrupts for each engine instance */
160 xe_mmio_write32(gt, RCS0_RSVD_INTR_MASK, ~smask);
161 xe_mmio_write32(gt, BCS_RSVD_INTR_MASK, ~smask);
162 if (bcs_mask & (BIT(1)|BIT(2)))
163 xe_mmio_write32(gt, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask);
164 if (bcs_mask & (BIT(3)|BIT(4)))
165 xe_mmio_write32(gt, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask);
166 if (bcs_mask & (BIT(5)|BIT(6)))
167 xe_mmio_write32(gt, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask);
168 if (bcs_mask & (BIT(7)|BIT(8)))
169 xe_mmio_write32(gt, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask);
170 if (ccs_mask & (BIT(0)|BIT(1)))
171 xe_mmio_write32(gt, CCS0_CCS1_INTR_MASK, ~dmask);
172 if (ccs_mask & (BIT(2)|BIT(3)))
173 xe_mmio_write32(gt, CCS2_CCS3_INTR_MASK, ~dmask);
174 }
175
176 if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) {
177 /* Enable interrupts for each engine class */
178 xe_mmio_write32(gt, VCS_VECS_INTR_ENABLE, dmask);
179
180 /* Unmask interrupts for each engine instance */
181 xe_mmio_write32(gt, VCS0_VCS1_INTR_MASK, ~dmask);
182 xe_mmio_write32(gt, VCS2_VCS3_INTR_MASK, ~dmask);
183 xe_mmio_write32(gt, VECS0_VECS1_INTR_MASK, ~dmask);
184
185 /*
186 * the heci2 interrupt is enabled via the same register as the
187 * GSCCS interrupts, but it has its own mask register.
188 */
189 if (xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER)) {
190 gsc_mask = irqs;
191 heci_mask = GSC_IRQ_INTF(1);
192 } else if (HAS_HECI_GSCFI(xe)) {
193 gsc_mask = GSC_IRQ_INTF(1);
194 }
195
196 if (gsc_mask) {
197 xe_mmio_write32(gt, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask);
198 xe_mmio_write32(gt, GUNIT_GSC_INTR_MASK, ~gsc_mask);
199 }
200 if (heci_mask)
201 xe_mmio_write32(gt, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16));
202 }
203}
204
205static u32
206gt_engine_identity(struct xe_device *xe,
207 struct xe_gt *mmio,
208 const unsigned int bank,
209 const unsigned int bit)
210{
211 u32 timeout_ts;
212 u32 ident;
213
214 lockdep_assert_held(&xe->irq.lock);
215
216 xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit));
217
218 /*
219 * NB: Specs do not specify how long to spin wait,
220 * so we do ~100us as an educated guess.
221 */
222 timeout_ts = (local_clock() >> 10) + 100;
223 do {
224 ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank));
225 } while (!(ident & INTR_DATA_VALID) &&
226 !time_after32(local_clock() >> 10, timeout_ts));
227
228 if (unlikely(!(ident & INTR_DATA_VALID))) {
229 drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
230 bank, bit, ident);
231 return 0;
232 }
233
234 xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident);
235
236 return ident;
237}
238
239#define OTHER_MEDIA_GUC_INSTANCE 16
240
241static void
242gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir)
243{
244 if (instance == OTHER_GUC_INSTANCE && !xe_gt_is_media_type(gt))
245 return xe_guc_irq_handler(>->uc.guc, iir);
246 if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt))
247 return xe_guc_irq_handler(>->uc.guc, iir);
248 if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt))
249 return xe_gsc_proxy_irq_handler(>->uc.gsc, iir);
250
251 if (instance != OTHER_GUC_INSTANCE &&
252 instance != OTHER_MEDIA_GUC_INSTANCE) {
253 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
254 instance, iir);
255 }
256}
257
258static struct xe_gt *pick_engine_gt(struct xe_tile *tile,
259 enum xe_engine_class class,
260 unsigned int instance)
261{
262 struct xe_device *xe = tile_to_xe(tile);
263
264 if (MEDIA_VER(xe) < 13)
265 return tile->primary_gt;
266
267 switch (class) {
268 case XE_ENGINE_CLASS_VIDEO_DECODE:
269 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
270 return tile->media_gt;
271 case XE_ENGINE_CLASS_OTHER:
272 switch (instance) {
273 case OTHER_MEDIA_GUC_INSTANCE:
274 case OTHER_GSC_INSTANCE:
275 case OTHER_GSC_HECI2_INSTANCE:
276 return tile->media_gt;
277 default:
278 break;
279 };
280 fallthrough;
281 default:
282 return tile->primary_gt;
283 }
284}
285
286static void gt_irq_handler(struct xe_tile *tile,
287 u32 master_ctl, unsigned long *intr_dw,
288 u32 *identity)
289{
290 struct xe_device *xe = tile_to_xe(tile);
291 struct xe_gt *mmio = tile->primary_gt;
292 unsigned int bank, bit;
293 u16 instance, intr_vec;
294 enum xe_engine_class class;
295 struct xe_hw_engine *hwe;
296
297 spin_lock(&xe->irq.lock);
298
299 for (bank = 0; bank < 2; bank++) {
300 if (!(master_ctl & GT_DW_IRQ(bank)))
301 continue;
302
303 intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank));
304 for_each_set_bit(bit, intr_dw + bank, 32)
305 identity[bit] = gt_engine_identity(xe, mmio, bank, bit);
306 xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]);
307
308 for_each_set_bit(bit, intr_dw + bank, 32) {
309 struct xe_gt *engine_gt;
310
311 class = INTR_ENGINE_CLASS(identity[bit]);
312 instance = INTR_ENGINE_INSTANCE(identity[bit]);
313 intr_vec = INTR_ENGINE_INTR(identity[bit]);
314
315 engine_gt = pick_engine_gt(tile, class, instance);
316
317 hwe = xe_gt_hw_engine(engine_gt, class, instance, false);
318 if (hwe) {
319 xe_hw_engine_handle_irq(hwe, intr_vec);
320 continue;
321 }
322
323 if (class == XE_ENGINE_CLASS_OTHER) {
324 /* HECI GSCFI interrupts come from outside of GT */
325 if (HAS_HECI_GSCFI(xe) && instance == OTHER_GSC_INSTANCE)
326 xe_heci_gsc_irq_handler(xe, intr_vec);
327 else
328 gt_other_irq_handler(engine_gt, instance, intr_vec);
329 continue;
330 }
331 }
332 }
333
334 spin_unlock(&xe->irq.lock);
335}
336
337/*
338 * Top-level interrupt handler for Xe_LP platforms (which did not have
339 * a "master tile" interrupt register.
340 */
341static irqreturn_t xelp_irq_handler(int irq, void *arg)
342{
343 struct xe_device *xe = arg;
344 struct xe_tile *tile = xe_device_get_root_tile(xe);
345 u32 master_ctl, gu_misc_iir;
346 unsigned long intr_dw[2];
347 u32 identity[32];
348
349 spin_lock(&xe->irq.lock);
350 if (!xe->irq.enabled) {
351 spin_unlock(&xe->irq.lock);
352 return IRQ_NONE;
353 }
354 spin_unlock(&xe->irq.lock);
355
356 master_ctl = xelp_intr_disable(xe);
357 if (!master_ctl) {
358 xelp_intr_enable(xe, false);
359 return IRQ_NONE;
360 }
361
362 gt_irq_handler(tile, master_ctl, intr_dw, identity);
363
364 xe_display_irq_handler(xe, master_ctl);
365
366 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
367
368 xelp_intr_enable(xe, false);
369
370 xe_display_irq_enable(xe, gu_misc_iir);
371
372 return IRQ_HANDLED;
373}
374
375static u32 dg1_intr_disable(struct xe_device *xe)
376{
377 struct xe_gt *mmio = xe_root_mmio_gt(xe);
378 u32 val;
379
380 /* First disable interrupts */
381 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0);
382
383 /* Get the indication levels and ack the master unit */
384 val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
385 if (unlikely(!val))
386 return 0;
387
388 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val);
389
390 return val;
391}
392
393static void dg1_intr_enable(struct xe_device *xe, bool stall)
394{
395 struct xe_gt *mmio = xe_root_mmio_gt(xe);
396
397 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
398 if (stall)
399 xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
400}
401
402/*
403 * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have
404 * a "master tile" interrupt register which must be consulted before the
405 * "graphics master" interrupt register.
406 */
407static irqreturn_t dg1_irq_handler(int irq, void *arg)
408{
409 struct xe_device *xe = arg;
410 struct xe_tile *tile;
411 u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0;
412 unsigned long intr_dw[2];
413 u32 identity[32];
414 u8 id;
415
416 /* TODO: This really shouldn't be copied+pasted */
417
418 spin_lock(&xe->irq.lock);
419 if (!xe->irq.enabled) {
420 spin_unlock(&xe->irq.lock);
421 return IRQ_NONE;
422 }
423 spin_unlock(&xe->irq.lock);
424
425 master_tile_ctl = dg1_intr_disable(xe);
426 if (!master_tile_ctl) {
427 dg1_intr_enable(xe, false);
428 return IRQ_NONE;
429 }
430
431 for_each_tile(tile, xe, id) {
432 struct xe_gt *mmio = tile->primary_gt;
433
434 if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0)
435 continue;
436
437 master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ);
438
439 /*
440 * We might be in irq handler just when PCIe DPC is initiated
441 * and all MMIO reads will be returned with all 1's. Ignore this
442 * irq as device is inaccessible.
443 */
444 if (master_ctl == REG_GENMASK(31, 0)) {
445 drm_dbg(&tile_to_xe(tile)->drm,
446 "Ignore this IRQ as device might be in DPC containment.\n");
447 return IRQ_HANDLED;
448 }
449
450 xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl);
451
452 gt_irq_handler(tile, master_ctl, intr_dw, identity);
453
454 /*
455 * Display interrupts (including display backlight operations
456 * that get reported as Gunit GSE) would only be hooked up to
457 * the primary tile.
458 */
459 if (id == 0) {
460 xe_display_irq_handler(xe, master_ctl);
461 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
462 }
463 }
464
465 dg1_intr_enable(xe, false);
466 xe_display_irq_enable(xe, gu_misc_iir);
467
468 return IRQ_HANDLED;
469}
470
471static void gt_irq_reset(struct xe_tile *tile)
472{
473 struct xe_gt *mmio = tile->primary_gt;
474
475 u32 ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
476 XE_ENGINE_CLASS_COMPUTE);
477 u32 bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
478 XE_ENGINE_CLASS_COPY);
479
480 /* Disable RCS, BCS, VCS and VECS class engines. */
481 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0);
482 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0);
483 if (ccs_mask)
484 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0);
485
486 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
487 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0);
488 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0);
489 if (bcs_mask & (BIT(1)|BIT(2)))
490 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
491 if (bcs_mask & (BIT(3)|BIT(4)))
492 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
493 if (bcs_mask & (BIT(5)|BIT(6)))
494 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
495 if (bcs_mask & (BIT(7)|BIT(8)))
496 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
497 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0);
498 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0);
499 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0);
500 if (ccs_mask & (BIT(0)|BIT(1)))
501 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0);
502 if (ccs_mask & (BIT(2)|BIT(3)))
503 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0);
504
505 if ((tile->media_gt &&
506 xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) ||
507 HAS_HECI_GSCFI(tile_to_xe(tile))) {
508 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0);
509 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0);
510 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0);
511 }
512
513 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0);
514 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0);
515 xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0);
516 xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0);
517}
518
519static void xelp_irq_reset(struct xe_tile *tile)
520{
521 xelp_intr_disable(tile_to_xe(tile));
522
523 gt_irq_reset(tile);
524
525 if (IS_SRIOV_VF(tile_to_xe(tile)))
526 return;
527
528 mask_and_disable(tile, PCU_IRQ_OFFSET);
529}
530
531static void dg1_irq_reset(struct xe_tile *tile)
532{
533 if (tile->id == 0)
534 dg1_intr_disable(tile_to_xe(tile));
535
536 gt_irq_reset(tile);
537
538 if (IS_SRIOV_VF(tile_to_xe(tile)))
539 return;
540
541 mask_and_disable(tile, PCU_IRQ_OFFSET);
542}
543
544static void dg1_irq_reset_mstr(struct xe_tile *tile)
545{
546 struct xe_gt *mmio = tile->primary_gt;
547
548 xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0);
549}
550
551static void vf_irq_reset(struct xe_device *xe)
552{
553 struct xe_tile *tile;
554 unsigned int id;
555
556 xe_assert(xe, IS_SRIOV_VF(xe));
557
558 if (GRAPHICS_VERx100(xe) < 1210)
559 xelp_intr_disable(xe);
560 else
561 xe_assert(xe, xe_device_has_memirq(xe));
562
563 for_each_tile(tile, xe, id) {
564 if (xe_device_has_memirq(xe))
565 xe_memirq_reset(&tile->sriov.vf.memirq);
566 else
567 gt_irq_reset(tile);
568 }
569}
570
571static void xe_irq_reset(struct xe_device *xe)
572{
573 struct xe_tile *tile;
574 u8 id;
575
576 if (IS_SRIOV_VF(xe))
577 return vf_irq_reset(xe);
578
579 for_each_tile(tile, xe, id) {
580 if (GRAPHICS_VERx100(xe) >= 1210)
581 dg1_irq_reset(tile);
582 else
583 xelp_irq_reset(tile);
584 }
585
586 tile = xe_device_get_root_tile(xe);
587 mask_and_disable(tile, GU_MISC_IRQ_OFFSET);
588 xe_display_irq_reset(xe);
589
590 /*
591 * The tile's top-level status register should be the last one
592 * to be reset to avoid possible bit re-latching from lower
593 * level interrupts.
594 */
595 if (GRAPHICS_VERx100(xe) >= 1210) {
596 for_each_tile(tile, xe, id)
597 dg1_irq_reset_mstr(tile);
598 }
599}
600
601static void vf_irq_postinstall(struct xe_device *xe)
602{
603 struct xe_tile *tile;
604 unsigned int id;
605
606 for_each_tile(tile, xe, id)
607 if (xe_device_has_memirq(xe))
608 xe_memirq_postinstall(&tile->sriov.vf.memirq);
609
610 if (GRAPHICS_VERx100(xe) < 1210)
611 xelp_intr_enable(xe, true);
612 else
613 xe_assert(xe, xe_device_has_memirq(xe));
614}
615
616static void xe_irq_postinstall(struct xe_device *xe)
617{
618 if (IS_SRIOV_VF(xe))
619 return vf_irq_postinstall(xe);
620
621 xe_display_irq_postinstall(xe, xe_root_mmio_gt(xe));
622
623 /*
624 * ASLE backlight operations are reported via GUnit GSE interrupts
625 * on the root tile.
626 */
627 unmask_and_enable(xe_device_get_root_tile(xe),
628 GU_MISC_IRQ_OFFSET, GU_MISC_GSE);
629
630 /* Enable top-level interrupts */
631 if (GRAPHICS_VERx100(xe) >= 1210)
632 dg1_intr_enable(xe, true);
633 else
634 xelp_intr_enable(xe, true);
635}
636
637static irqreturn_t vf_mem_irq_handler(int irq, void *arg)
638{
639 struct xe_device *xe = arg;
640 struct xe_tile *tile;
641 unsigned int id;
642
643 spin_lock(&xe->irq.lock);
644 if (!xe->irq.enabled) {
645 spin_unlock(&xe->irq.lock);
646 return IRQ_NONE;
647 }
648 spin_unlock(&xe->irq.lock);
649
650 for_each_tile(tile, xe, id)
651 xe_memirq_handler(&tile->sriov.vf.memirq);
652
653 return IRQ_HANDLED;
654}
655
656static irq_handler_t xe_irq_handler(struct xe_device *xe)
657{
658 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe))
659 return vf_mem_irq_handler;
660
661 if (GRAPHICS_VERx100(xe) >= 1210)
662 return dg1_irq_handler;
663 else
664 return xelp_irq_handler;
665}
666
667static void irq_uninstall(struct drm_device *drm, void *arg)
668{
669 struct xe_device *xe = arg;
670 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
671 int irq;
672
673 if (!xe->irq.enabled)
674 return;
675
676 xe->irq.enabled = false;
677 xe_irq_reset(xe);
678
679 irq = pci_irq_vector(pdev, 0);
680 free_irq(irq, xe);
681}
682
683int xe_irq_install(struct xe_device *xe)
684{
685 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
686 unsigned int irq_flags = PCI_IRQ_MSIX;
687 irq_handler_t irq_handler;
688 int err, irq, nvec;
689
690 irq_handler = xe_irq_handler(xe);
691 if (!irq_handler) {
692 drm_err(&xe->drm, "No supported interrupt handler");
693 return -EINVAL;
694 }
695
696 xe_irq_reset(xe);
697
698 nvec = pci_msix_vec_count(pdev);
699 if (nvec <= 0) {
700 if (nvec == -EINVAL) {
701 /* MSIX capability is not supported in the device, using MSI */
702 irq_flags = PCI_IRQ_MSI;
703 nvec = 1;
704 } else {
705 drm_err(&xe->drm, "MSIX: Failed getting count\n");
706 return nvec;
707 }
708 }
709
710 err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags);
711 if (err < 0) {
712 drm_err(&xe->drm, "MSI/MSIX: Failed to enable support %d\n", err);
713 return err;
714 }
715
716 irq = pci_irq_vector(pdev, 0);
717 err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe);
718 if (err < 0) {
719 drm_err(&xe->drm, "Failed to request MSI/MSIX IRQ %d\n", err);
720 return err;
721 }
722
723 xe->irq.enabled = true;
724
725 xe_irq_postinstall(xe);
726
727 err = drmm_add_action_or_reset(&xe->drm, irq_uninstall, xe);
728 if (err)
729 goto free_irq_handler;
730
731 return 0;
732
733free_irq_handler:
734 free_irq(irq, xe);
735
736 return err;
737}
738
739void xe_irq_shutdown(struct xe_device *xe)
740{
741 irq_uninstall(&xe->drm, xe);
742}
743
744void xe_irq_suspend(struct xe_device *xe)
745{
746 int irq = to_pci_dev(xe->drm.dev)->irq;
747
748 spin_lock_irq(&xe->irq.lock);
749 xe->irq.enabled = false; /* no new irqs */
750 spin_unlock_irq(&xe->irq.lock);
751
752 synchronize_irq(irq); /* flush irqs */
753 xe_irq_reset(xe); /* turn irqs off */
754}
755
756void xe_irq_resume(struct xe_device *xe)
757{
758 struct xe_gt *gt;
759 int id;
760
761 /*
762 * lock not needed:
763 * 1. no irq will arrive before the postinstall
764 * 2. display is not yet resumed
765 */
766 xe->irq.enabled = true;
767 xe_irq_reset(xe);
768 xe_irq_postinstall(xe); /* turn irqs on */
769
770 for_each_gt(gt, xe, id)
771 xe_irq_enable_hwe(gt);
772}