Loading...
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <linux/sched/clock.h>
7
8#include "i915_drv.h"
9#include "i915_irq.h"
10#include "intel_breadcrumbs.h"
11#include "intel_gt.h"
12#include "intel_gt_irq.h"
13#include "intel_lrc_reg.h"
14#include "intel_uncore.h"
15#include "intel_rps.h"
16
17static void guc_irq_handler(struct intel_guc *guc, u16 iir)
18{
19 if (iir & GUC_INTR_GUC2HOST)
20 intel_guc_to_host_event_handler(guc);
21}
22
23static u32
24gen11_gt_engine_identity(struct intel_gt *gt,
25 const unsigned int bank, const unsigned int bit)
26{
27 void __iomem * const regs = gt->uncore->regs;
28 u32 timeout_ts;
29 u32 ident;
30
31 lockdep_assert_held(>->irq_lock);
32
33 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
34
35 /*
36 * NB: Specs do not specify how long to spin wait,
37 * so we do ~100us as an educated guess.
38 */
39 timeout_ts = (local_clock() >> 10) + 100;
40 do {
41 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
42 } while (!(ident & GEN11_INTR_DATA_VALID) &&
43 !time_after32(local_clock() >> 10, timeout_ts));
44
45 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
46 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
47 bank, bit, ident);
48 return 0;
49 }
50
51 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
52 GEN11_INTR_DATA_VALID);
53
54 return ident;
55}
56
57static void
58gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
59 const u16 iir)
60{
61 if (instance == OTHER_GUC_INSTANCE)
62 return guc_irq_handler(>->uc.guc, iir);
63
64 if (instance == OTHER_GTPM_INSTANCE)
65 return gen11_rps_irq_handler(>->rps, iir);
66
67 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
68 instance, iir);
69}
70
71static void
72gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
73 const u8 instance, const u16 iir)
74{
75 struct intel_engine_cs *engine;
76
77 if (instance <= MAX_ENGINE_INSTANCE)
78 engine = gt->engine_class[class][instance];
79 else
80 engine = NULL;
81
82 if (likely(engine))
83 return intel_engine_cs_irq(engine, iir);
84
85 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
86 class, instance);
87}
88
89static void
90gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
91{
92 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
93 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
94 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
95
96 if (unlikely(!intr))
97 return;
98
99 if (class <= COPY_ENGINE_CLASS)
100 return gen11_engine_irq_handler(gt, class, instance, intr);
101
102 if (class == OTHER_CLASS)
103 return gen11_other_irq_handler(gt, instance, intr);
104
105 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
106 class, instance, intr);
107}
108
109static void
110gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
111{
112 void __iomem * const regs = gt->uncore->regs;
113 unsigned long intr_dw;
114 unsigned int bit;
115
116 lockdep_assert_held(>->irq_lock);
117
118 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
119
120 for_each_set_bit(bit, &intr_dw, 32) {
121 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
122
123 gen11_gt_identity_handler(gt, ident);
124 }
125
126 /* Clear must be after shared has been served for engine */
127 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
128}
129
130void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
131{
132 unsigned int bank;
133
134 spin_lock(>->irq_lock);
135
136 for (bank = 0; bank < 2; bank++) {
137 if (master_ctl & GEN11_GT_DW_IRQ(bank))
138 gen11_gt_bank_handler(gt, bank);
139 }
140
141 spin_unlock(>->irq_lock);
142}
143
144bool gen11_gt_reset_one_iir(struct intel_gt *gt,
145 const unsigned int bank, const unsigned int bit)
146{
147 void __iomem * const regs = gt->uncore->regs;
148 u32 dw;
149
150 lockdep_assert_held(>->irq_lock);
151
152 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
153 if (dw & BIT(bit)) {
154 /*
155 * According to the BSpec, DW_IIR bits cannot be cleared without
156 * first servicing the Selector & Shared IIR registers.
157 */
158 gen11_gt_engine_identity(gt, bank, bit);
159
160 /*
161 * We locked GT INT DW by reading it. If we want to (try
162 * to) recover from this successfully, we need to clear
163 * our bit, otherwise we are locking the register for
164 * everybody.
165 */
166 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
167
168 return true;
169 }
170
171 return false;
172}
173
174void gen11_gt_irq_reset(struct intel_gt *gt)
175{
176 struct intel_uncore *uncore = gt->uncore;
177
178 /* Disable RCS, BCS, VCS and VECS class engines. */
179 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
180 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0);
181
182 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
183 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0);
184 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0);
185 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0);
186 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0);
187 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
188
189 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
190 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
191 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
192 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
193}
194
195void gen11_gt_irq_postinstall(struct intel_gt *gt)
196{
197 struct intel_uncore *uncore = gt->uncore;
198 u32 irqs = GT_RENDER_USER_INTERRUPT;
199 u32 dmask;
200 u32 smask;
201
202 if (!intel_uc_wants_guc_submission(>->uc))
203 irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
204 GT_CONTEXT_SWITCH_INTERRUPT |
205 GT_WAIT_SEMAPHORE_INTERRUPT;
206
207 dmask = irqs << 16 | irqs;
208 smask = irqs << 16;
209
210 BUILD_BUG_ON(irqs & 0xffff0000);
211
212 /* Enable RCS, BCS, VCS and VECS class interrupts. */
213 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
214 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
215
216 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
217 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
218 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
219 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
220 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
221 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
222
223 /*
224 * RPS interrupts will get enabled/disabled on demand when RPS itself
225 * is enabled/disabled.
226 */
227 gt->pm_ier = 0x0;
228 gt->pm_imr = ~gt->pm_ier;
229 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
230 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
231
232 /* Same thing for GuC interrupts */
233 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
234 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
235}
236
237void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
238{
239 if (gt_iir & GT_RENDER_USER_INTERRUPT)
240 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
241 gt_iir);
242
243 if (gt_iir & ILK_BSD_USER_INTERRUPT)
244 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
245 gt_iir);
246}
247
248static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
249{
250 if (!HAS_L3_DPF(gt->i915))
251 return;
252
253 spin_lock(>->irq_lock);
254 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
255 spin_unlock(>->irq_lock);
256
257 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
258 gt->i915->l3_parity.which_slice |= 1 << 1;
259
260 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
261 gt->i915->l3_parity.which_slice |= 1 << 0;
262
263 schedule_work(>->i915->l3_parity.error_work);
264}
265
266void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
267{
268 if (gt_iir & GT_RENDER_USER_INTERRUPT)
269 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
270 gt_iir);
271
272 if (gt_iir & GT_BSD_USER_INTERRUPT)
273 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
274 gt_iir >> 12);
275
276 if (gt_iir & GT_BLT_USER_INTERRUPT)
277 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
278 gt_iir >> 22);
279
280 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
281 GT_BSD_CS_ERROR_INTERRUPT |
282 GT_CS_MASTER_ERROR_INTERRUPT))
283 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
284
285 if (gt_iir & GT_PARITY_ERROR(gt->i915))
286 gen7_parity_error_irq_handler(gt, gt_iir);
287}
288
289void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
290{
291 void __iomem * const regs = gt->uncore->regs;
292 u32 iir;
293
294 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
295 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
296 if (likely(iir)) {
297 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
298 iir >> GEN8_RCS_IRQ_SHIFT);
299 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
300 iir >> GEN8_BCS_IRQ_SHIFT);
301 raw_reg_write(regs, GEN8_GT_IIR(0), iir);
302 }
303 }
304
305 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
306 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
307 if (likely(iir)) {
308 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
309 iir >> GEN8_VCS0_IRQ_SHIFT);
310 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
311 iir >> GEN8_VCS1_IRQ_SHIFT);
312 raw_reg_write(regs, GEN8_GT_IIR(1), iir);
313 }
314 }
315
316 if (master_ctl & GEN8_GT_VECS_IRQ) {
317 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
318 if (likely(iir)) {
319 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
320 iir >> GEN8_VECS_IRQ_SHIFT);
321 raw_reg_write(regs, GEN8_GT_IIR(3), iir);
322 }
323 }
324
325 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
326 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
327 if (likely(iir)) {
328 gen6_rps_irq_handler(>->rps, iir);
329 guc_irq_handler(>->uc.guc, iir >> 16);
330 raw_reg_write(regs, GEN8_GT_IIR(2), iir);
331 }
332 }
333}
334
335void gen8_gt_irq_reset(struct intel_gt *gt)
336{
337 struct intel_uncore *uncore = gt->uncore;
338
339 GEN8_IRQ_RESET_NDX(uncore, GT, 0);
340 GEN8_IRQ_RESET_NDX(uncore, GT, 1);
341 GEN8_IRQ_RESET_NDX(uncore, GT, 2);
342 GEN8_IRQ_RESET_NDX(uncore, GT, 3);
343}
344
345void gen8_gt_irq_postinstall(struct intel_gt *gt)
346{
347 /* These are interrupts we'll toggle with the ring mask register */
348 const u32 irqs =
349 GT_CS_MASTER_ERROR_INTERRUPT |
350 GT_RENDER_USER_INTERRUPT |
351 GT_CONTEXT_SWITCH_INTERRUPT |
352 GT_WAIT_SEMAPHORE_INTERRUPT;
353 const u32 gt_interrupts[] = {
354 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
355 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
356 0,
357 irqs << GEN8_VECS_IRQ_SHIFT,
358 };
359 struct intel_uncore *uncore = gt->uncore;
360
361 gt->pm_ier = 0x0;
362 gt->pm_imr = ~gt->pm_ier;
363 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
364 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
365 /*
366 * RPS interrupts will get enabled/disabled on demand when RPS itself
367 * is enabled/disabled. Same wil be the case for GuC interrupts.
368 */
369 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
370 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
371}
372
373static void gen5_gt_update_irq(struct intel_gt *gt,
374 u32 interrupt_mask,
375 u32 enabled_irq_mask)
376{
377 lockdep_assert_held(>->irq_lock);
378
379 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
380
381 gt->gt_imr &= ~interrupt_mask;
382 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
383 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
384}
385
386void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
387{
388 gen5_gt_update_irq(gt, mask, mask);
389 intel_uncore_posting_read_fw(gt->uncore, GTIMR);
390}
391
392void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
393{
394 gen5_gt_update_irq(gt, mask, 0);
395}
396
397void gen5_gt_irq_reset(struct intel_gt *gt)
398{
399 struct intel_uncore *uncore = gt->uncore;
400
401 GEN3_IRQ_RESET(uncore, GT);
402 if (GRAPHICS_VER(gt->i915) >= 6)
403 GEN3_IRQ_RESET(uncore, GEN6_PM);
404}
405
406void gen5_gt_irq_postinstall(struct intel_gt *gt)
407{
408 struct intel_uncore *uncore = gt->uncore;
409 u32 pm_irqs = 0;
410 u32 gt_irqs = 0;
411
412 gt->gt_imr = ~0;
413 if (HAS_L3_DPF(gt->i915)) {
414 /* L3 parity interrupt is always unmasked. */
415 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
416 gt_irqs |= GT_PARITY_ERROR(gt->i915);
417 }
418
419 gt_irqs |= GT_RENDER_USER_INTERRUPT;
420 if (GRAPHICS_VER(gt->i915) == 5)
421 gt_irqs |= ILK_BSD_USER_INTERRUPT;
422 else
423 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
424
425 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
426
427 if (GRAPHICS_VER(gt->i915) >= 6) {
428 /*
429 * RPS interrupts will get enabled/disabled on demand when RPS
430 * itself is enabled/disabled.
431 */
432 if (HAS_ENGINE(gt, VECS0)) {
433 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
434 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
435 }
436
437 gt->pm_imr = 0xffffffff;
438 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
439 }
440}
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <linux/sched/clock.h>
7
8#include "i915_drv.h"
9#include "i915_irq.h"
10#include "i915_reg.h"
11#include "intel_breadcrumbs.h"
12#include "intel_gt.h"
13#include "intel_gt_irq.h"
14#include "intel_gt_print.h"
15#include "intel_gt_regs.h"
16#include "intel_uncore.h"
17#include "intel_rps.h"
18#include "pxp/intel_pxp_irq.h"
19#include "uc/intel_gsc_proxy.h"
20
21static void guc_irq_handler(struct intel_guc *guc, u16 iir)
22{
23 if (unlikely(!guc->interrupts.enabled))
24 return;
25
26 if (iir & GUC_INTR_GUC2HOST)
27 intel_guc_to_host_event_handler(guc);
28}
29
30static u32
31gen11_gt_engine_identity(struct intel_gt *gt,
32 const unsigned int bank, const unsigned int bit)
33{
34 void __iomem * const regs = intel_uncore_regs(gt->uncore);
35 u32 timeout_ts;
36 u32 ident;
37
38 lockdep_assert_held(gt->irq_lock);
39
40 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
41
42 /*
43 * NB: Specs do not specify how long to spin wait,
44 * so we do ~100us as an educated guess.
45 */
46 timeout_ts = (local_clock() >> 10) + 100;
47 do {
48 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
49 } while (!(ident & GEN11_INTR_DATA_VALID) &&
50 !time_after32(local_clock() >> 10, timeout_ts));
51
52 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
53 gt_err(gt, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
54 bank, bit, ident);
55 return 0;
56 }
57
58 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
59 GEN11_INTR_DATA_VALID);
60
61 return ident;
62}
63
64static void
65gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
66 const u16 iir)
67{
68 struct intel_gt *media_gt = gt->i915->media_gt;
69
70 if (instance == OTHER_GUC_INSTANCE)
71 return guc_irq_handler(>->uc.guc, iir);
72 if (instance == OTHER_MEDIA_GUC_INSTANCE && media_gt)
73 return guc_irq_handler(&media_gt->uc.guc, iir);
74
75 if (instance == OTHER_GTPM_INSTANCE)
76 return gen11_rps_irq_handler(>->rps, iir);
77 if (instance == OTHER_MEDIA_GTPM_INSTANCE && media_gt)
78 return gen11_rps_irq_handler(&media_gt->rps, iir);
79
80 if (instance == OTHER_KCR_INSTANCE)
81 return intel_pxp_irq_handler(gt->i915->pxp, iir);
82
83 if (instance == OTHER_GSC_INSTANCE)
84 return intel_gsc_irq_handler(gt, iir);
85
86 if (instance == OTHER_GSC_HECI_2_INSTANCE)
87 return intel_gsc_proxy_irq_handler(>->uc.gsc, iir);
88
89 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
90 instance, iir);
91}
92
93static struct intel_gt *pick_gt(struct intel_gt *gt, u8 class, u8 instance)
94{
95 struct intel_gt *media_gt = gt->i915->media_gt;
96
97 /* we expect the non-media gt to be passed in */
98 GEM_BUG_ON(gt == media_gt);
99
100 if (!media_gt)
101 return gt;
102
103 switch (class) {
104 case VIDEO_DECODE_CLASS:
105 case VIDEO_ENHANCEMENT_CLASS:
106 return media_gt;
107 case OTHER_CLASS:
108 if (instance == OTHER_GSC_HECI_2_INSTANCE)
109 return media_gt;
110 if ((instance == OTHER_GSC_INSTANCE || instance == OTHER_KCR_INSTANCE) &&
111 HAS_ENGINE(media_gt, GSC0))
112 return media_gt;
113 fallthrough;
114 default:
115 return gt;
116 }
117}
118
119static void
120gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
121{
122 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
123 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
124 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
125
126 if (unlikely(!intr))
127 return;
128
129 /*
130 * Platforms with standalone media have the media and GSC engines in
131 * another GT.
132 */
133 gt = pick_gt(gt, class, instance);
134
135 if (class <= MAX_ENGINE_CLASS && instance <= MAX_ENGINE_INSTANCE) {
136 struct intel_engine_cs *engine = gt->engine_class[class][instance];
137 if (engine)
138 return intel_engine_cs_irq(engine, intr);
139 }
140
141 if (class == OTHER_CLASS)
142 return gen11_other_irq_handler(gt, instance, intr);
143
144 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
145 class, instance, intr);
146}
147
148static void
149gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
150{
151 void __iomem * const regs = intel_uncore_regs(gt->uncore);
152 unsigned long intr_dw;
153 unsigned int bit;
154
155 lockdep_assert_held(gt->irq_lock);
156
157 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
158
159 for_each_set_bit(bit, &intr_dw, 32) {
160 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
161
162 gen11_gt_identity_handler(gt, ident);
163 }
164
165 /* Clear must be after shared has been served for engine */
166 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
167}
168
169void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
170{
171 unsigned int bank;
172
173 spin_lock(gt->irq_lock);
174
175 for (bank = 0; bank < 2; bank++) {
176 if (master_ctl & GEN11_GT_DW_IRQ(bank))
177 gen11_gt_bank_handler(gt, bank);
178 }
179
180 spin_unlock(gt->irq_lock);
181}
182
183bool gen11_gt_reset_one_iir(struct intel_gt *gt,
184 const unsigned int bank, const unsigned int bit)
185{
186 void __iomem * const regs = intel_uncore_regs(gt->uncore);
187 u32 dw;
188
189 lockdep_assert_held(gt->irq_lock);
190
191 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
192 if (dw & BIT(bit)) {
193 /*
194 * According to the BSpec, DW_IIR bits cannot be cleared without
195 * first servicing the Selector & Shared IIR registers.
196 */
197 gen11_gt_engine_identity(gt, bank, bit);
198
199 /*
200 * We locked GT INT DW by reading it. If we want to (try
201 * to) recover from this successfully, we need to clear
202 * our bit, otherwise we are locking the register for
203 * everybody.
204 */
205 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
206
207 return true;
208 }
209
210 return false;
211}
212
213void gen11_gt_irq_reset(struct intel_gt *gt)
214{
215 struct intel_uncore *uncore = gt->uncore;
216
217 /* Disable RCS, BCS, VCS and VECS class engines. */
218 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
219 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0);
220 if (CCS_MASK(gt))
221 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0);
222 if (HAS_HECI_GSC(gt->i915) || HAS_ENGINE(gt, GSC0))
223 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, 0);
224
225 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
226 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0);
227 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0);
228 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
229 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
230 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
231 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
232 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
233 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
234 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
235 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
236 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0);
237 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0);
238 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
239 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0);
240 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
241 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0);
242 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
243 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
244 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0);
245 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
246 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0);
247 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
248 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0);
249 if (HAS_HECI_GSC(gt->i915) || HAS_ENGINE(gt, GSC0))
250 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~0);
251
252 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
253 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
254 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
255 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
256
257 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, 0);
258 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~0);
259}
260
261void gen11_gt_irq_postinstall(struct intel_gt *gt)
262{
263 struct intel_uncore *uncore = gt->uncore;
264 u32 irqs = GT_RENDER_USER_INTERRUPT;
265 u32 guc_mask = intel_uc_wants_guc(>->uc) ? GUC_INTR_GUC2HOST : 0;
266 u32 gsc_mask = 0;
267 u32 heci_mask = 0;
268 u32 dmask;
269 u32 smask;
270
271 if (!intel_uc_wants_guc_submission(>->uc))
272 irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
273 GT_CONTEXT_SWITCH_INTERRUPT |
274 GT_WAIT_SEMAPHORE_INTERRUPT;
275
276 dmask = irqs << 16 | irqs;
277 smask = irqs << 16;
278
279 if (HAS_ENGINE(gt, GSC0)) {
280 /*
281 * the heci2 interrupt is enabled via the same register as the
282 * GSC interrupt, but it has its own mask register.
283 */
284 gsc_mask = irqs;
285 heci_mask = GSC_IRQ_INTF(1); /* HECI2 IRQ for SW Proxy*/
286 } else if (HAS_HECI_GSC(gt->i915)) {
287 gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1);
288 }
289
290 BUILD_BUG_ON(irqs & 0xffff0000);
291
292 /* Enable RCS, BCS, VCS and VECS class interrupts. */
293 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
294 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
295 if (CCS_MASK(gt))
296 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask);
297 if (gsc_mask)
298 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, gsc_mask | heci_mask);
299
300 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
301 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
302 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
303 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
304 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask);
305 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
306 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask);
307 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
308 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask);
309 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
310 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask);
311 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
312 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
313 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
314 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask);
315 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
316 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask);
317 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
318 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
319 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask);
320 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
321 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask);
322 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
323 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask);
324 if (gsc_mask)
325 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~gsc_mask);
326 if (heci_mask)
327 intel_uncore_write(uncore, GEN12_HECI2_RSVD_INTR_MASK,
328 ~REG_FIELD_PREP(ENGINE1_MASK, heci_mask));
329
330 if (guc_mask) {
331 /* the enable bit is common for both GTs but the masks are separate */
332 u32 mask = gt->type == GT_MEDIA ?
333 REG_FIELD_PREP(ENGINE0_MASK, guc_mask) :
334 REG_FIELD_PREP(ENGINE1_MASK, guc_mask);
335
336 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE,
337 REG_FIELD_PREP(ENGINE1_MASK, guc_mask));
338
339 /* we might not be the first GT to write this reg */
340 intel_uncore_rmw(uncore, MTL_GUC_MGUC_INTR_MASK, mask, 0);
341 }
342
343 /*
344 * RPS interrupts will get enabled/disabled on demand when RPS itself
345 * is enabled/disabled.
346 */
347 gt->pm_ier = 0x0;
348 gt->pm_imr = ~gt->pm_ier;
349 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
350 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
351}
352
353void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
354{
355 if (gt_iir & GT_RENDER_USER_INTERRUPT)
356 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
357 gt_iir);
358
359 if (gt_iir & ILK_BSD_USER_INTERRUPT)
360 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
361 gt_iir);
362}
363
364static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
365{
366 if (!HAS_L3_DPF(gt->i915))
367 return;
368
369 spin_lock(gt->irq_lock);
370 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
371 spin_unlock(gt->irq_lock);
372
373 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
374 gt->i915->l3_parity.which_slice |= 1 << 1;
375
376 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
377 gt->i915->l3_parity.which_slice |= 1 << 0;
378
379 queue_work(gt->i915->unordered_wq, >->i915->l3_parity.error_work);
380}
381
382void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
383{
384 if (gt_iir & GT_RENDER_USER_INTERRUPT)
385 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
386 gt_iir);
387
388 if (gt_iir & GT_BSD_USER_INTERRUPT)
389 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
390 gt_iir >> 12);
391
392 if (gt_iir & GT_BLT_USER_INTERRUPT)
393 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
394 gt_iir >> 22);
395
396 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
397 GT_BSD_CS_ERROR_INTERRUPT |
398 GT_CS_MASTER_ERROR_INTERRUPT))
399 gt_dbg(gt, "Command parser error, gt_iir 0x%08x\n", gt_iir);
400
401 if (gt_iir & GT_PARITY_ERROR(gt->i915))
402 gen7_parity_error_irq_handler(gt, gt_iir);
403}
404
405void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
406{
407 void __iomem * const regs = intel_uncore_regs(gt->uncore);
408 u32 iir;
409
410 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
411 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
412 if (likely(iir)) {
413 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
414 iir >> GEN8_RCS_IRQ_SHIFT);
415 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
416 iir >> GEN8_BCS_IRQ_SHIFT);
417 raw_reg_write(regs, GEN8_GT_IIR(0), iir);
418 }
419 }
420
421 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
422 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
423 if (likely(iir)) {
424 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
425 iir >> GEN8_VCS0_IRQ_SHIFT);
426 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
427 iir >> GEN8_VCS1_IRQ_SHIFT);
428 raw_reg_write(regs, GEN8_GT_IIR(1), iir);
429 }
430 }
431
432 if (master_ctl & GEN8_GT_VECS_IRQ) {
433 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
434 if (likely(iir)) {
435 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
436 iir >> GEN8_VECS_IRQ_SHIFT);
437 raw_reg_write(regs, GEN8_GT_IIR(3), iir);
438 }
439 }
440
441 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
442 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
443 if (likely(iir)) {
444 gen6_rps_irq_handler(>->rps, iir);
445 guc_irq_handler(>->uc.guc, iir >> 16);
446 raw_reg_write(regs, GEN8_GT_IIR(2), iir);
447 }
448 }
449}
450
451void gen8_gt_irq_reset(struct intel_gt *gt)
452{
453 struct intel_uncore *uncore = gt->uncore;
454
455 GEN8_IRQ_RESET_NDX(uncore, GT, 0);
456 GEN8_IRQ_RESET_NDX(uncore, GT, 1);
457 GEN8_IRQ_RESET_NDX(uncore, GT, 2);
458 GEN8_IRQ_RESET_NDX(uncore, GT, 3);
459}
460
461void gen8_gt_irq_postinstall(struct intel_gt *gt)
462{
463 /* These are interrupts we'll toggle with the ring mask register */
464 const u32 irqs =
465 GT_CS_MASTER_ERROR_INTERRUPT |
466 GT_RENDER_USER_INTERRUPT |
467 GT_CONTEXT_SWITCH_INTERRUPT |
468 GT_WAIT_SEMAPHORE_INTERRUPT;
469 const u32 gt_interrupts[] = {
470 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
471 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
472 0,
473 irqs << GEN8_VECS_IRQ_SHIFT,
474 };
475 struct intel_uncore *uncore = gt->uncore;
476
477 gt->pm_ier = 0x0;
478 gt->pm_imr = ~gt->pm_ier;
479 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
480 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
481 /*
482 * RPS interrupts will get enabled/disabled on demand when RPS itself
483 * is enabled/disabled. Same wil be the case for GuC interrupts.
484 */
485 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
486 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
487}
488
489static void gen5_gt_update_irq(struct intel_gt *gt,
490 u32 interrupt_mask,
491 u32 enabled_irq_mask)
492{
493 lockdep_assert_held(gt->irq_lock);
494
495 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
496
497 gt->gt_imr &= ~interrupt_mask;
498 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
499 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
500}
501
502void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
503{
504 gen5_gt_update_irq(gt, mask, mask);
505 intel_uncore_posting_read_fw(gt->uncore, GTIMR);
506}
507
508void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
509{
510 gen5_gt_update_irq(gt, mask, 0);
511}
512
513void gen5_gt_irq_reset(struct intel_gt *gt)
514{
515 struct intel_uncore *uncore = gt->uncore;
516
517 GEN3_IRQ_RESET(uncore, GT);
518 if (GRAPHICS_VER(gt->i915) >= 6)
519 GEN3_IRQ_RESET(uncore, GEN6_PM);
520}
521
522void gen5_gt_irq_postinstall(struct intel_gt *gt)
523{
524 struct intel_uncore *uncore = gt->uncore;
525 u32 pm_irqs = 0;
526 u32 gt_irqs = 0;
527
528 gt->gt_imr = ~0;
529 if (HAS_L3_DPF(gt->i915)) {
530 /* L3 parity interrupt is always unmasked. */
531 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
532 gt_irqs |= GT_PARITY_ERROR(gt->i915);
533 }
534
535 gt_irqs |= GT_RENDER_USER_INTERRUPT;
536 if (GRAPHICS_VER(gt->i915) == 5)
537 gt_irqs |= ILK_BSD_USER_INTERRUPT;
538 else
539 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
540
541 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
542
543 if (GRAPHICS_VER(gt->i915) >= 6) {
544 /*
545 * RPS interrupts will get enabled/disabled on demand when RPS
546 * itself is enabled/disabled.
547 */
548 if (HAS_ENGINE(gt, VECS0)) {
549 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
550 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
551 }
552
553 gt->pm_imr = 0xffffffff;
554 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
555 }
556}