Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2014-2018 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "intel_context.h"
8#include "intel_engine_pm.h"
9#include "intel_gpu_commands.h"
10#include "intel_gt.h"
11#include "intel_ring.h"
12#include "intel_workarounds.h"
13
14/**
15 * DOC: Hardware workarounds
16 *
17 * This file is intended as a central place to implement most [1]_ of the
18 * required workarounds for hardware to work as originally intended. They fall
19 * in five basic categories depending on how/when they are applied:
20 *
21 * - Workarounds that touch registers that are saved/restored to/from the HW
22 * context image. The list is emitted (via Load Register Immediate commands)
23 * everytime a new context is created.
24 * - GT workarounds. The list of these WAs is applied whenever these registers
25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..).
26 * - Display workarounds. The list is applied during display clock-gating
27 * initialization.
28 * - Workarounds that whitelist a privileged register, so that UMDs can manage
29 * them directly. This is just a special case of a MMMIO workaround (as we
30 * write the list of these to/be-whitelisted registers to some special HW
31 * registers).
32 * - Workaround batchbuffers, that get executed automatically by the hardware
33 * on every HW context restore.
34 *
35 * .. [1] Please notice that there are other WAs that, due to their nature,
36 * cannot be applied from a central place. Those are peppered around the rest
37 * of the code, as needed.
38 *
39 * .. [2] Technically, some registers are powercontext saved & restored, so they
40 * survive a suspend/resume. In practice, writing them again is not too
41 * costly and simplifies things. We can revisit this in the future.
42 *
43 * Layout
44 * ~~~~~~
45 *
46 * Keep things in this file ordered by WA type, as per the above (context, GT,
47 * display, register whitelist, batchbuffer). Then, inside each type, keep the
48 * following order:
49 *
50 * - Infrastructure functions and macros
51 * - WAs per platform in standard gen/chrono order
52 * - Public functions to init or apply the given workaround type.
53 */
54
55static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
56{
57 wal->name = name;
58 wal->engine_name = engine_name;
59}
60
61#define WA_LIST_CHUNK (1 << 4)
62
63static void wa_init_finish(struct i915_wa_list *wal)
64{
65 /* Trim unused entries. */
66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
67 struct i915_wa *list = kmemdup(wal->list,
68 wal->count * sizeof(*list),
69 GFP_KERNEL);
70
71 if (list) {
72 kfree(wal->list);
73 wal->list = list;
74 }
75 }
76
77 if (!wal->count)
78 return;
79
80 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
81 wal->wa_count, wal->name, wal->engine_name);
82}
83
84static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
85{
86 unsigned int addr = i915_mmio_reg_offset(wa->reg);
87 unsigned int start = 0, end = wal->count;
88 const unsigned int grow = WA_LIST_CHUNK;
89 struct i915_wa *wa_;
90
91 GEM_BUG_ON(!is_power_of_2(grow));
92
93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
94 struct i915_wa *list;
95
96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
97 GFP_KERNEL);
98 if (!list) {
99 DRM_ERROR("No space for workaround init!\n");
100 return;
101 }
102
103 if (wal->list) {
104 memcpy(list, wal->list, sizeof(*wa) * wal->count);
105 kfree(wal->list);
106 }
107
108 wal->list = list;
109 }
110
111 while (start < end) {
112 unsigned int mid = start + (end - start) / 2;
113
114 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
115 start = mid + 1;
116 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
117 end = mid;
118 } else {
119 wa_ = &wal->list[mid];
120
121 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
122 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
123 i915_mmio_reg_offset(wa_->reg),
124 wa_->clr, wa_->set);
125
126 wa_->set &= ~wa->clr;
127 }
128
129 wal->wa_count++;
130 wa_->set |= wa->set;
131 wa_->clr |= wa->clr;
132 wa_->read |= wa->read;
133 return;
134 }
135 }
136
137 wal->wa_count++;
138 wa_ = &wal->list[wal->count++];
139 *wa_ = *wa;
140
141 while (wa_-- > wal->list) {
142 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
143 i915_mmio_reg_offset(wa_[1].reg));
144 if (i915_mmio_reg_offset(wa_[1].reg) >
145 i915_mmio_reg_offset(wa_[0].reg))
146 break;
147
148 swap(wa_[1], wa_[0]);
149 }
150}
151
152static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
153 u32 clear, u32 set, u32 read_mask)
154{
155 struct i915_wa wa = {
156 .reg = reg,
157 .clr = clear,
158 .set = set,
159 .read = read_mask,
160 };
161
162 _wa_add(wal, &wa);
163}
164
165static void
166wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
167{
168 wa_add(wal, reg, clear, set, clear);
169}
170
171static void
172wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
173{
174 wa_write_clr_set(wal, reg, ~0, set);
175}
176
177static void
178wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
179{
180 wa_write_clr_set(wal, reg, set, set);
181}
182
183static void
184wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
185{
186 wa_write_clr_set(wal, reg, clr, 0);
187}
188
189/*
190 * WA operations on "masked register". A masked register has the upper 16 bits
191 * documented as "masked" in b-spec. Its purpose is to allow writing to just a
192 * portion of the register without a rmw: you simply write in the upper 16 bits
193 * the mask of bits you are going to modify.
194 *
195 * The wa_masked_* family of functions already does the necessary operations to
196 * calculate the mask based on the parameters passed, so user only has to
197 * provide the lower 16 bits of that register.
198 */
199
200static void
201wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
202{
203 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
204}
205
206static void
207wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
208{
209 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
210}
211
212static void
213wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
214 u32 mask, u32 val)
215{
216 wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask);
217}
218
219static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
220 struct i915_wa_list *wal)
221{
222 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
223}
224
225static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
226 struct i915_wa_list *wal)
227{
228 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
229}
230
231static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
232 struct i915_wa_list *wal)
233{
234 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
235
236 /* WaDisableAsyncFlipPerfMode:bdw,chv */
237 wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE);
238
239 /* WaDisablePartialInstShootdown:bdw,chv */
240 wa_masked_en(wal, GEN8_ROW_CHICKEN,
241 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
242
243 /* Use Force Non-Coherent whenever executing a 3D context. This is a
244 * workaround for a possible hang in the unlikely event a TLB
245 * invalidation occurs during a PSD flush.
246 */
247 /* WaForceEnableNonCoherent:bdw,chv */
248 /* WaHdcDisableFetchWhenMasked:bdw,chv */
249 wa_masked_en(wal, HDC_CHICKEN0,
250 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
251 HDC_FORCE_NON_COHERENT);
252
253 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
254 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
255 * polygons in the same 8x4 pixel/sample area to be processed without
256 * stalling waiting for the earlier ones to write to Hierarchical Z
257 * buffer."
258 *
259 * This optimization is off by default for BDW and CHV; turn it on.
260 */
261 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
262
263 /* Wa4x4STCOptimizationDisable:bdw,chv */
264 wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
265
266 /*
267 * BSpec recommends 8x4 when MSAA is used,
268 * however in practice 16x4 seems fastest.
269 *
270 * Note that PS/WM thread counts depend on the WIZ hashing
271 * disable bit, which we don't touch here, but it's good
272 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
273 */
274 wa_masked_field_set(wal, GEN7_GT_MODE,
275 GEN6_WIZ_HASHING_MASK,
276 GEN6_WIZ_HASHING_16x4);
277}
278
279static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
280 struct i915_wa_list *wal)
281{
282 struct drm_i915_private *i915 = engine->i915;
283
284 gen8_ctx_workarounds_init(engine, wal);
285
286 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
287 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
288
289 /* WaDisableDopClockGating:bdw
290 *
291 * Also see the related UCGTCL1 write in bdw_init_clock_gating()
292 * to disable EUTC clock gating.
293 */
294 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
295 DOP_CLOCK_GATING_DISABLE);
296
297 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
298 GEN8_SAMPLER_POWER_BYPASS_DIS);
299
300 wa_masked_en(wal, HDC_CHICKEN0,
301 /* WaForceContextSaveRestoreNonCoherent:bdw */
302 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
303 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
304 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
305}
306
307static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
308 struct i915_wa_list *wal)
309{
310 gen8_ctx_workarounds_init(engine, wal);
311
312 /* WaDisableThreadStallDopClockGating:chv */
313 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
314
315 /* Improve HiZ throughput on CHV. */
316 wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
317}
318
319static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
320 struct i915_wa_list *wal)
321{
322 struct drm_i915_private *i915 = engine->i915;
323
324 if (HAS_LLC(i915)) {
325 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
326 *
327 * Must match Display Engine. See
328 * WaCompressedResourceDisplayNewHashMode.
329 */
330 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
331 GEN9_PBE_COMPRESSED_HASH_SELECTION);
332 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
333 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
334 }
335
336 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
337 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
338 wa_masked_en(wal, GEN8_ROW_CHICKEN,
339 FLOW_CONTROL_ENABLE |
340 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
341
342 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
343 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
344 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
345 GEN9_ENABLE_YV12_BUGFIX |
346 GEN9_ENABLE_GPGPU_PREEMPTION);
347
348 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
349 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
350 wa_masked_en(wal, CACHE_MODE_1,
351 GEN8_4x4_STC_OPTIMIZATION_DISABLE |
352 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
353
354 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
355 wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
356 GEN9_CCS_TLB_PREFETCH_ENABLE);
357
358 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
359 wa_masked_en(wal, HDC_CHICKEN0,
360 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
361 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
362
363 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
364 * both tied to WaForceContextSaveRestoreNonCoherent
365 * in some hsds for skl. We keep the tie for all gen9. The
366 * documentation is a bit hazy and so we want to get common behaviour,
367 * even though there is no clear evidence we would need both on kbl/bxt.
368 * This area has been source of system hangs so we play it safe
369 * and mimic the skl regardless of what bspec says.
370 *
371 * Use Force Non-Coherent whenever executing a 3D context. This
372 * is a workaround for a possible hang in the unlikely event
373 * a TLB invalidation occurs during a PSD flush.
374 */
375
376 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
377 wa_masked_en(wal, HDC_CHICKEN0,
378 HDC_FORCE_NON_COHERENT);
379
380 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
381 if (IS_SKYLAKE(i915) ||
382 IS_KABYLAKE(i915) ||
383 IS_COFFEELAKE(i915) ||
384 IS_COMETLAKE(i915))
385 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
386 GEN8_SAMPLER_POWER_BYPASS_DIS);
387
388 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
389 wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
390
391 /*
392 * Supporting preemption with fine-granularity requires changes in the
393 * batch buffer programming. Since we can't break old userspace, we
394 * need to set our default preemption level to safe value. Userspace is
395 * still able to use more fine-grained preemption levels, since in
396 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
397 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
398 * not real HW workarounds, but merely a way to start using preemption
399 * while maintaining old contract with userspace.
400 */
401
402 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
403 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
404
405 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
406 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
407 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
408 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
409
410 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
411 if (IS_GEN9_LP(i915))
412 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
413}
414
415static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
416 struct i915_wa_list *wal)
417{
418 struct intel_gt *gt = engine->gt;
419 u8 vals[3] = { 0, 0, 0 };
420 unsigned int i;
421
422 for (i = 0; i < 3; i++) {
423 u8 ss;
424
425 /*
426 * Only consider slices where one, and only one, subslice has 7
427 * EUs
428 */
429 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
430 continue;
431
432 /*
433 * subslice_7eu[i] != 0 (because of the check above) and
434 * ss_max == 4 (maximum number of subslices possible per slice)
435 *
436 * -> 0 <= ss <= 3;
437 */
438 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
439 vals[i] = 3 - ss;
440 }
441
442 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
443 return;
444
445 /* Tune IZ hashing. See intel_device_info_runtime_init() */
446 wa_masked_field_set(wal, GEN7_GT_MODE,
447 GEN9_IZ_HASHING_MASK(2) |
448 GEN9_IZ_HASHING_MASK(1) |
449 GEN9_IZ_HASHING_MASK(0),
450 GEN9_IZ_HASHING(2, vals[2]) |
451 GEN9_IZ_HASHING(1, vals[1]) |
452 GEN9_IZ_HASHING(0, vals[0]));
453}
454
455static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
456 struct i915_wa_list *wal)
457{
458 gen9_ctx_workarounds_init(engine, wal);
459 skl_tune_iz_hashing(engine, wal);
460}
461
462static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
463 struct i915_wa_list *wal)
464{
465 gen9_ctx_workarounds_init(engine, wal);
466
467 /* WaDisableThreadStallDopClockGating:bxt */
468 wa_masked_en(wal, GEN8_ROW_CHICKEN,
469 STALL_DOP_GATING_DISABLE);
470
471 /* WaToEnableHwFixForPushConstHWBug:bxt */
472 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
473 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
474}
475
476static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
477 struct i915_wa_list *wal)
478{
479 struct drm_i915_private *i915 = engine->i915;
480
481 gen9_ctx_workarounds_init(engine, wal);
482
483 /* WaToEnableHwFixForPushConstHWBug:kbl */
484 if (IS_KBL_GT_STEP(i915, STEP_C0, STEP_FOREVER))
485 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
486 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
487
488 /* WaDisableSbeCacheDispatchPortSharing:kbl */
489 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
490 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
491}
492
493static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
494 struct i915_wa_list *wal)
495{
496 gen9_ctx_workarounds_init(engine, wal);
497
498 /* WaToEnableHwFixForPushConstHWBug:glk */
499 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
500 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
501}
502
503static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
504 struct i915_wa_list *wal)
505{
506 gen9_ctx_workarounds_init(engine, wal);
507
508 /* WaToEnableHwFixForPushConstHWBug:cfl */
509 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
510 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
511
512 /* WaDisableSbeCacheDispatchPortSharing:cfl */
513 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
514 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
515}
516
517static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
518 struct i915_wa_list *wal)
519{
520 /* WaForceContextSaveRestoreNonCoherent:cnl */
521 wa_masked_en(wal, CNL_HDC_CHICKEN0,
522 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
523
524 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
525 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
526 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
527
528 /* WaPushConstantDereferenceHoldDisable:cnl */
529 wa_masked_en(wal, GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
530
531 /* FtrEnableFastAnisoL1BankingFix:cnl */
532 wa_masked_en(wal, HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
533
534 /* WaDisable3DMidCmdPreemption:cnl */
535 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
536
537 /* WaDisableGPGPUMidCmdPreemption:cnl */
538 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
539 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
540 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
541
542 /* WaDisableEarlyEOT:cnl */
543 wa_masked_en(wal, GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
544}
545
546static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
547 struct i915_wa_list *wal)
548{
549 struct drm_i915_private *i915 = engine->i915;
550
551 /* WaDisableBankHangMode:icl */
552 wa_write(wal,
553 GEN8_L3CNTLREG,
554 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
555 GEN8_ERRDETBCTRL);
556
557 /* Wa_1604370585:icl (pre-prod)
558 * Formerly known as WaPushConstantDereferenceHoldDisable
559 */
560 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
561 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
562 PUSH_CONSTANT_DEREF_DISABLE);
563
564 /* WaForceEnableNonCoherent:icl
565 * This is not the same workaround as in early Gen9 platforms, where
566 * lacking this could cause system hangs, but coherency performance
567 * overhead is high and only a few compute workloads really need it
568 * (the register is whitelisted in hardware now, so UMDs can opt in
569 * for coherency if they have a good reason).
570 */
571 wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
572
573 /* Wa_2006611047:icl (pre-prod)
574 * Formerly known as WaDisableImprovedTdlClkGating
575 */
576 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
577 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
578 GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
579
580 /* Wa_2006665173:icl (pre-prod) */
581 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
582 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
583 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
584
585 /* WaEnableFloatBlendOptimization:icl */
586 wa_write_clr_set(wal,
587 GEN10_CACHE_MODE_SS,
588 0, /* write-only, so skip validation */
589 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
590
591 /* WaDisableGPGPUMidThreadPreemption:icl */
592 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
593 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
594 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
595
596 /* allow headerless messages for preemptible GPGPU context */
597 wa_masked_en(wal, GEN10_SAMPLER_MODE,
598 GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
599
600 /* Wa_1604278689:icl,ehl */
601 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
602 wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
603 0, /* write-only register; skip validation */
604 0xFFFFFFFF);
605
606 /* Wa_1406306137:icl,ehl */
607 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
608}
609
610/*
611 * These settings aren't actually workarounds, but general tuning settings that
612 * need to be programmed on several platforms.
613 */
614static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
615 struct i915_wa_list *wal)
616{
617 /*
618 * Although some platforms refer to it as Wa_1604555607, we need to
619 * program it even on those that don't explicitly list that
620 * workaround.
621 *
622 * Note that the programming of this register is further modified
623 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
624 * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
625 * value when read. The default value for this register is zero for all
626 * fields and there are no bit masks. So instead of doing a RMW we
627 * should just write TDS timer value. For the same reason read
628 * verification is ignored.
629 */
630 wa_add(wal,
631 FF_MODE2,
632 FF_MODE2_TDS_TIMER_MASK,
633 FF_MODE2_TDS_TIMER_128,
634 0);
635}
636
637static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
638 struct i915_wa_list *wal)
639{
640 gen12_ctx_gt_tuning_init(engine, wal);
641
642 /*
643 * Wa_1409142259:tgl
644 * Wa_1409347922:tgl
645 * Wa_1409252684:tgl
646 * Wa_1409217633:tgl
647 * Wa_1409207793:tgl
648 * Wa_1409178076:tgl
649 * Wa_1408979724:tgl
650 * Wa_14010443199:rkl
651 * Wa_14010698770:rkl
652 */
653 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
654 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
655
656 /* WaDisableGPGPUMidThreadPreemption:gen12 */
657 wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
658 GEN9_PREEMPT_GPGPU_LEVEL_MASK,
659 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
660
661 /*
662 * Wa_16011163337
663 *
664 * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
665 * to Wa_1608008084.
666 */
667 wa_add(wal,
668 FF_MODE2,
669 FF_MODE2_GS_TIMER_MASK,
670 FF_MODE2_GS_TIMER_224,
671 0);
672}
673
674static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
675 struct i915_wa_list *wal)
676{
677 gen12_ctx_workarounds_init(engine, wal);
678
679 /* Wa_1409044764 */
680 wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
681 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
682
683 /* Wa_22010493298 */
684 wa_masked_en(wal, HIZ_CHICKEN,
685 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
686}
687
688static void
689__intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
690 struct i915_wa_list *wal,
691 const char *name)
692{
693 struct drm_i915_private *i915 = engine->i915;
694
695 if (engine->class != RENDER_CLASS)
696 return;
697
698 wa_init_start(wal, name, engine->name);
699
700 if (IS_DG1(i915))
701 dg1_ctx_workarounds_init(engine, wal);
702 else if (GRAPHICS_VER(i915) == 12)
703 gen12_ctx_workarounds_init(engine, wal);
704 else if (GRAPHICS_VER(i915) == 11)
705 icl_ctx_workarounds_init(engine, wal);
706 else if (IS_CANNONLAKE(i915))
707 cnl_ctx_workarounds_init(engine, wal);
708 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
709 cfl_ctx_workarounds_init(engine, wal);
710 else if (IS_GEMINILAKE(i915))
711 glk_ctx_workarounds_init(engine, wal);
712 else if (IS_KABYLAKE(i915))
713 kbl_ctx_workarounds_init(engine, wal);
714 else if (IS_BROXTON(i915))
715 bxt_ctx_workarounds_init(engine, wal);
716 else if (IS_SKYLAKE(i915))
717 skl_ctx_workarounds_init(engine, wal);
718 else if (IS_CHERRYVIEW(i915))
719 chv_ctx_workarounds_init(engine, wal);
720 else if (IS_BROADWELL(i915))
721 bdw_ctx_workarounds_init(engine, wal);
722 else if (GRAPHICS_VER(i915) == 7)
723 gen7_ctx_workarounds_init(engine, wal);
724 else if (GRAPHICS_VER(i915) == 6)
725 gen6_ctx_workarounds_init(engine, wal);
726 else if (GRAPHICS_VER(i915) < 8)
727 ;
728 else
729 MISSING_CASE(GRAPHICS_VER(i915));
730
731 wa_init_finish(wal);
732}
733
734void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
735{
736 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
737}
738
739int intel_engine_emit_ctx_wa(struct i915_request *rq)
740{
741 struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
742 struct i915_wa *wa;
743 unsigned int i;
744 u32 *cs;
745 int ret;
746
747 if (wal->count == 0)
748 return 0;
749
750 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
751 if (ret)
752 return ret;
753
754 cs = intel_ring_begin(rq, (wal->count * 2 + 2));
755 if (IS_ERR(cs))
756 return PTR_ERR(cs);
757
758 *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
759 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
760 *cs++ = i915_mmio_reg_offset(wa->reg);
761 *cs++ = wa->set;
762 }
763 *cs++ = MI_NOOP;
764
765 intel_ring_advance(rq, cs);
766
767 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
768 if (ret)
769 return ret;
770
771 return 0;
772}
773
774static void
775gen4_gt_workarounds_init(struct drm_i915_private *i915,
776 struct i915_wa_list *wal)
777{
778 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
779 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
780}
781
782static void
783g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
784{
785 gen4_gt_workarounds_init(i915, wal);
786
787 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
788 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
789}
790
791static void
792ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
793{
794 g4x_gt_workarounds_init(i915, wal);
795
796 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
797}
798
799static void
800snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
801{
802}
803
804static void
805ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
806{
807 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
808 wa_masked_dis(wal,
809 GEN7_COMMON_SLICE_CHICKEN1,
810 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
811
812 /* WaApplyL3ControlAndL3ChickenMode:ivb */
813 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
814 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
815
816 /* WaForceL3Serialization:ivb */
817 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
818}
819
820static void
821vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
822{
823 /* WaForceL3Serialization:vlv */
824 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
825
826 /*
827 * WaIncreaseL3CreditsForVLVB0:vlv
828 * This is the hardware default actually.
829 */
830 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
831}
832
833static void
834hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
835{
836 /* L3 caching of data atomics doesn't work -- disable it. */
837 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
838
839 wa_add(wal,
840 HSW_ROW_CHICKEN3, 0,
841 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
842 0 /* XXX does this reg exist? */);
843
844 /* WaVSRefCountFullforceMissDisable:hsw */
845 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
846}
847
848static void
849gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
850{
851 /* WaDisableKillLogic:bxt,skl,kbl */
852 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
853 wa_write_or(wal,
854 GAM_ECOCHK,
855 ECOCHK_DIS_TLB);
856
857 if (HAS_LLC(i915)) {
858 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
859 *
860 * Must match Display Engine. See
861 * WaCompressedResourceDisplayNewHashMode.
862 */
863 wa_write_or(wal,
864 MMCD_MISC_CTRL,
865 MMCD_PCLA | MMCD_HOTSPOT_EN);
866 }
867
868 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
869 wa_write_or(wal,
870 GAM_ECOCHK,
871 BDW_DISABLE_HDC_INVALIDATION);
872}
873
874static void
875skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
876{
877 gen9_gt_workarounds_init(i915, wal);
878
879 /* WaDisableGafsUnitClkGating:skl */
880 wa_write_or(wal,
881 GEN7_UCGCTL4,
882 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
883
884 /* WaInPlaceDecompressionHang:skl */
885 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
886 wa_write_or(wal,
887 GEN9_GAMT_ECO_REG_RW_IA,
888 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
889}
890
891static void
892bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
893{
894 gen9_gt_workarounds_init(i915, wal);
895
896 /* WaInPlaceDecompressionHang:bxt */
897 wa_write_or(wal,
898 GEN9_GAMT_ECO_REG_RW_IA,
899 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
900}
901
902static void
903kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
904{
905 gen9_gt_workarounds_init(i915, wal);
906
907 /* WaDisableDynamicCreditSharing:kbl */
908 if (IS_KBL_GT_STEP(i915, 0, STEP_B0))
909 wa_write_or(wal,
910 GAMT_CHKN_BIT_REG,
911 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
912
913 /* WaDisableGafsUnitClkGating:kbl */
914 wa_write_or(wal,
915 GEN7_UCGCTL4,
916 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
917
918 /* WaInPlaceDecompressionHang:kbl */
919 wa_write_or(wal,
920 GEN9_GAMT_ECO_REG_RW_IA,
921 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
922}
923
924static void
925glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
926{
927 gen9_gt_workarounds_init(i915, wal);
928}
929
930static void
931cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
932{
933 gen9_gt_workarounds_init(i915, wal);
934
935 /* WaDisableGafsUnitClkGating:cfl */
936 wa_write_or(wal,
937 GEN7_UCGCTL4,
938 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
939
940 /* WaInPlaceDecompressionHang:cfl */
941 wa_write_or(wal,
942 GEN9_GAMT_ECO_REG_RW_IA,
943 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
944}
945
946static void
947wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
948{
949 const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
950 unsigned int slice, subslice;
951 u32 l3_en, mcr, mcr_mask;
952
953 GEM_BUG_ON(GRAPHICS_VER(i915) < 10);
954
955 /*
956 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
957 * L3Banks could be fused off in single slice scenario. If that is
958 * the case, we might need to program MCR select to a valid L3Bank
959 * by default, to make sure we correctly read certain registers
960 * later on (in the range 0xB100 - 0xB3FF).
961 *
962 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
963 * Before any MMIO read into slice/subslice specific registers, MCR
964 * packet control register needs to be programmed to point to any
965 * enabled s/ss pair. Otherwise, incorrect values will be returned.
966 * This means each subsequent MMIO read will be forwarded to an
967 * specific s/ss combination, but this is OK since these registers
968 * are consistent across s/ss in almost all cases. In the rare
969 * occasions, such as INSTDONE, where this value is dependent
970 * on s/ss combo, the read should be done with read_subslice_reg.
971 *
972 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
973 * to which subslice, or to which L3 bank, the respective mmio reads
974 * will go, we have to find a common index which works for both
975 * accesses.
976 *
977 * Case where we cannot find a common index fortunately should not
978 * happen in production hardware, so we only emit a warning instead of
979 * implementing something more complex that requires checking the range
980 * of every MMIO read.
981 */
982
983 if (GRAPHICS_VER(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
984 u32 l3_fuse =
985 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
986 GEN10_L3BANK_MASK;
987
988 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
989 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
990 } else {
991 l3_en = ~0;
992 }
993
994 slice = fls(sseu->slice_mask) - 1;
995 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
996 if (!subslice) {
997 drm_warn(&i915->drm,
998 "No common index found between subslice mask %x and L3 bank mask %x!\n",
999 intel_sseu_get_subslices(sseu, slice), l3_en);
1000 subslice = fls(l3_en);
1001 drm_WARN_ON(&i915->drm, !subslice);
1002 }
1003 subslice--;
1004
1005 if (GRAPHICS_VER(i915) >= 11) {
1006 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1007 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1008 } else {
1009 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1010 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1011 }
1012
1013 drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1014
1015 wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1016}
1017
1018static void
1019cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1020{
1021 wa_init_mcr(i915, wal);
1022
1023 /* WaInPlaceDecompressionHang:cnl */
1024 wa_write_or(wal,
1025 GEN9_GAMT_ECO_REG_RW_IA,
1026 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1027}
1028
1029static void
1030icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1031{
1032 wa_init_mcr(i915, wal);
1033
1034 /* WaInPlaceDecompressionHang:icl */
1035 wa_write_or(wal,
1036 GEN9_GAMT_ECO_REG_RW_IA,
1037 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1038
1039 /* WaModifyGamTlbPartitioning:icl */
1040 wa_write_clr_set(wal,
1041 GEN11_GACB_PERF_CTRL,
1042 GEN11_HASH_CTRL_MASK,
1043 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1044
1045 /* Wa_1405766107:icl
1046 * Formerly known as WaCL2SFHalfMaxAlloc
1047 */
1048 wa_write_or(wal,
1049 GEN11_LSN_UNSLCVC,
1050 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1051 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1052
1053 /* Wa_220166154:icl
1054 * Formerly known as WaDisCtxReload
1055 */
1056 wa_write_or(wal,
1057 GEN8_GAMW_ECO_DEV_RW_IA,
1058 GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1059
1060 /* Wa_1405779004:icl (pre-prod) */
1061 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1062 wa_write_or(wal,
1063 SLICE_UNIT_LEVEL_CLKGATE,
1064 MSCUNIT_CLKGATE_DIS);
1065
1066 /* Wa_1406838659:icl (pre-prod) */
1067 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1068 wa_write_or(wal,
1069 INF_UNIT_LEVEL_CLKGATE,
1070 CGPSF_CLKGATE_DIS);
1071
1072 /* Wa_1406463099:icl
1073 * Formerly known as WaGamTlbPendError
1074 */
1075 wa_write_or(wal,
1076 GAMT_CHKN_BIT_REG,
1077 GAMT_CHKN_DISABLE_L3_COH_PIPE);
1078
1079 /* Wa_1607087056:icl,ehl,jsl */
1080 if (IS_ICELAKE(i915) ||
1081 IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0))
1082 wa_write_or(wal,
1083 SLICE_UNIT_LEVEL_CLKGATE,
1084 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1085}
1086
1087/*
1088 * Though there are per-engine instances of these registers,
1089 * they retain their value through engine resets and should
1090 * only be provided on the GT workaround list rather than
1091 * the engine-specific workaround list.
1092 */
1093static void
1094wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal)
1095{
1096 struct intel_engine_cs *engine;
1097 struct intel_gt *gt = &i915->gt;
1098 int id;
1099
1100 for_each_engine(engine, gt, id) {
1101 if (engine->class != VIDEO_DECODE_CLASS ||
1102 (engine->instance % 2))
1103 continue;
1104
1105 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1106 IECPUNIT_CLKGATE_DIS);
1107 }
1108}
1109
1110static void
1111gen12_gt_workarounds_init(struct drm_i915_private *i915,
1112 struct i915_wa_list *wal)
1113{
1114 wa_init_mcr(i915, wal);
1115
1116 /* Wa_14011060649:tgl,rkl,dg1,adls */
1117 wa_14011060649(i915, wal);
1118}
1119
1120static void
1121tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1122{
1123 gen12_gt_workarounds_init(i915, wal);
1124
1125 /* Wa_1409420604:tgl */
1126 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1127 wa_write_or(wal,
1128 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1129 CPSSUNIT_CLKGATE_DIS);
1130
1131 /* Wa_1607087056:tgl also know as BUG:1409180338 */
1132 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1133 wa_write_or(wal,
1134 SLICE_UNIT_LEVEL_CLKGATE,
1135 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1136
1137 /* Wa_1408615072:tgl[a0] */
1138 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1139 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1140 VSUNIT_CLKGATE_DIS_TGL);
1141}
1142
1143static void
1144dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1145{
1146 gen12_gt_workarounds_init(i915, wal);
1147
1148 /* Wa_1607087056:dg1 */
1149 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0))
1150 wa_write_or(wal,
1151 SLICE_UNIT_LEVEL_CLKGATE,
1152 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1153
1154 /* Wa_1409420604:dg1 */
1155 if (IS_DG1(i915))
1156 wa_write_or(wal,
1157 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1158 CPSSUNIT_CLKGATE_DIS);
1159
1160 /* Wa_1408615072:dg1 */
1161 /* Empirical testing shows this register is unaffected by engine reset. */
1162 if (IS_DG1(i915))
1163 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1164 VSUNIT_CLKGATE_DIS_TGL);
1165}
1166
1167static void
1168gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1169{
1170 if (IS_DG1(i915))
1171 dg1_gt_workarounds_init(i915, wal);
1172 else if (IS_TIGERLAKE(i915))
1173 tgl_gt_workarounds_init(i915, wal);
1174 else if (GRAPHICS_VER(i915) == 12)
1175 gen12_gt_workarounds_init(i915, wal);
1176 else if (GRAPHICS_VER(i915) == 11)
1177 icl_gt_workarounds_init(i915, wal);
1178 else if (IS_CANNONLAKE(i915))
1179 cnl_gt_workarounds_init(i915, wal);
1180 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1181 cfl_gt_workarounds_init(i915, wal);
1182 else if (IS_GEMINILAKE(i915))
1183 glk_gt_workarounds_init(i915, wal);
1184 else if (IS_KABYLAKE(i915))
1185 kbl_gt_workarounds_init(i915, wal);
1186 else if (IS_BROXTON(i915))
1187 bxt_gt_workarounds_init(i915, wal);
1188 else if (IS_SKYLAKE(i915))
1189 skl_gt_workarounds_init(i915, wal);
1190 else if (IS_HASWELL(i915))
1191 hsw_gt_workarounds_init(i915, wal);
1192 else if (IS_VALLEYVIEW(i915))
1193 vlv_gt_workarounds_init(i915, wal);
1194 else if (IS_IVYBRIDGE(i915))
1195 ivb_gt_workarounds_init(i915, wal);
1196 else if (GRAPHICS_VER(i915) == 6)
1197 snb_gt_workarounds_init(i915, wal);
1198 else if (GRAPHICS_VER(i915) == 5)
1199 ilk_gt_workarounds_init(i915, wal);
1200 else if (IS_G4X(i915))
1201 g4x_gt_workarounds_init(i915, wal);
1202 else if (GRAPHICS_VER(i915) == 4)
1203 gen4_gt_workarounds_init(i915, wal);
1204 else if (GRAPHICS_VER(i915) <= 8)
1205 ;
1206 else
1207 MISSING_CASE(GRAPHICS_VER(i915));
1208}
1209
1210void intel_gt_init_workarounds(struct drm_i915_private *i915)
1211{
1212 struct i915_wa_list *wal = &i915->gt_wa_list;
1213
1214 wa_init_start(wal, "GT", "global");
1215 gt_init_workarounds(i915, wal);
1216 wa_init_finish(wal);
1217}
1218
1219static enum forcewake_domains
1220wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1221{
1222 enum forcewake_domains fw = 0;
1223 struct i915_wa *wa;
1224 unsigned int i;
1225
1226 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1227 fw |= intel_uncore_forcewake_for_reg(uncore,
1228 wa->reg,
1229 FW_REG_READ |
1230 FW_REG_WRITE);
1231
1232 return fw;
1233}
1234
1235static bool
1236wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1237{
1238 if ((cur ^ wa->set) & wa->read) {
1239 DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1240 name, from, i915_mmio_reg_offset(wa->reg),
1241 cur, cur & wa->read, wa->set & wa->read);
1242
1243 return false;
1244 }
1245
1246 return true;
1247}
1248
1249static void
1250wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1251{
1252 enum forcewake_domains fw;
1253 unsigned long flags;
1254 struct i915_wa *wa;
1255 unsigned int i;
1256
1257 if (!wal->count)
1258 return;
1259
1260 fw = wal_get_fw_for_rmw(uncore, wal);
1261
1262 spin_lock_irqsave(&uncore->lock, flags);
1263 intel_uncore_forcewake_get__locked(uncore, fw);
1264
1265 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1266 if (wa->clr)
1267 intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1268 else
1269 intel_uncore_write_fw(uncore, wa->reg, wa->set);
1270 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1271 wa_verify(wa,
1272 intel_uncore_read_fw(uncore, wa->reg),
1273 wal->name, "application");
1274 }
1275
1276 intel_uncore_forcewake_put__locked(uncore, fw);
1277 spin_unlock_irqrestore(&uncore->lock, flags);
1278}
1279
1280void intel_gt_apply_workarounds(struct intel_gt *gt)
1281{
1282 wa_list_apply(gt->uncore, >->i915->gt_wa_list);
1283}
1284
1285static bool wa_list_verify(struct intel_uncore *uncore,
1286 const struct i915_wa_list *wal,
1287 const char *from)
1288{
1289 struct i915_wa *wa;
1290 unsigned int i;
1291 bool ok = true;
1292
1293 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1294 ok &= wa_verify(wa,
1295 intel_uncore_read(uncore, wa->reg),
1296 wal->name, from);
1297
1298 return ok;
1299}
1300
1301bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1302{
1303 return wa_list_verify(gt->uncore, >->i915->gt_wa_list, from);
1304}
1305
1306__maybe_unused
1307static bool is_nonpriv_flags_valid(u32 flags)
1308{
1309 /* Check only valid flag bits are set */
1310 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1311 return false;
1312
1313 /* NB: Only 3 out of 4 enum values are valid for access field */
1314 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1315 RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1316 return false;
1317
1318 return true;
1319}
1320
1321static void
1322whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1323{
1324 struct i915_wa wa = {
1325 .reg = reg
1326 };
1327
1328 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1329 return;
1330
1331 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1332 return;
1333
1334 wa.reg.reg |= flags;
1335 _wa_add(wal, &wa);
1336}
1337
1338static void
1339whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1340{
1341 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1342}
1343
1344static void gen9_whitelist_build(struct i915_wa_list *w)
1345{
1346 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1347 whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1348
1349 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1350 whitelist_reg(w, GEN8_CS_CHICKEN1);
1351
1352 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1353 whitelist_reg(w, GEN8_HDC_CHICKEN1);
1354
1355 /* WaSendPushConstantsFromMMIO:skl,bxt */
1356 whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1357}
1358
1359static void skl_whitelist_build(struct intel_engine_cs *engine)
1360{
1361 struct i915_wa_list *w = &engine->whitelist;
1362
1363 if (engine->class != RENDER_CLASS)
1364 return;
1365
1366 gen9_whitelist_build(w);
1367
1368 /* WaDisableLSQCROPERFforOCL:skl */
1369 whitelist_reg(w, GEN8_L3SQCREG4);
1370}
1371
1372static void bxt_whitelist_build(struct intel_engine_cs *engine)
1373{
1374 if (engine->class != RENDER_CLASS)
1375 return;
1376
1377 gen9_whitelist_build(&engine->whitelist);
1378}
1379
1380static void kbl_whitelist_build(struct intel_engine_cs *engine)
1381{
1382 struct i915_wa_list *w = &engine->whitelist;
1383
1384 if (engine->class != RENDER_CLASS)
1385 return;
1386
1387 gen9_whitelist_build(w);
1388
1389 /* WaDisableLSQCROPERFforOCL:kbl */
1390 whitelist_reg(w, GEN8_L3SQCREG4);
1391}
1392
1393static void glk_whitelist_build(struct intel_engine_cs *engine)
1394{
1395 struct i915_wa_list *w = &engine->whitelist;
1396
1397 if (engine->class != RENDER_CLASS)
1398 return;
1399
1400 gen9_whitelist_build(w);
1401
1402 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1403 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1404}
1405
1406static void cfl_whitelist_build(struct intel_engine_cs *engine)
1407{
1408 struct i915_wa_list *w = &engine->whitelist;
1409
1410 if (engine->class != RENDER_CLASS)
1411 return;
1412
1413 gen9_whitelist_build(w);
1414
1415 /*
1416 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1417 *
1418 * This covers 4 register which are next to one another :
1419 * - PS_INVOCATION_COUNT
1420 * - PS_INVOCATION_COUNT_UDW
1421 * - PS_DEPTH_COUNT
1422 * - PS_DEPTH_COUNT_UDW
1423 */
1424 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1425 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1426 RING_FORCE_TO_NONPRIV_RANGE_4);
1427}
1428
1429static void cml_whitelist_build(struct intel_engine_cs *engine)
1430{
1431 struct i915_wa_list *w = &engine->whitelist;
1432
1433 if (engine->class != RENDER_CLASS)
1434 whitelist_reg_ext(w,
1435 RING_CTX_TIMESTAMP(engine->mmio_base),
1436 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1437
1438 cfl_whitelist_build(engine);
1439}
1440
1441static void cnl_whitelist_build(struct intel_engine_cs *engine)
1442{
1443 struct i915_wa_list *w = &engine->whitelist;
1444
1445 if (engine->class != RENDER_CLASS)
1446 return;
1447
1448 /* WaEnablePreemptionGranularityControlByUMD:cnl */
1449 whitelist_reg(w, GEN8_CS_CHICKEN1);
1450}
1451
1452static void icl_whitelist_build(struct intel_engine_cs *engine)
1453{
1454 struct i915_wa_list *w = &engine->whitelist;
1455
1456 switch (engine->class) {
1457 case RENDER_CLASS:
1458 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1459 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1460
1461 /* WaAllowUMDToModifySamplerMode:icl */
1462 whitelist_reg(w, GEN10_SAMPLER_MODE);
1463
1464 /* WaEnableStateCacheRedirectToCS:icl */
1465 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1466
1467 /*
1468 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1469 *
1470 * This covers 4 register which are next to one another :
1471 * - PS_INVOCATION_COUNT
1472 * - PS_INVOCATION_COUNT_UDW
1473 * - PS_DEPTH_COUNT
1474 * - PS_DEPTH_COUNT_UDW
1475 */
1476 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1477 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1478 RING_FORCE_TO_NONPRIV_RANGE_4);
1479 break;
1480
1481 case VIDEO_DECODE_CLASS:
1482 /* hucStatusRegOffset */
1483 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1484 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1485 /* hucUKernelHdrInfoRegOffset */
1486 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1487 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1488 /* hucStatus2RegOffset */
1489 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1490 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1491 whitelist_reg_ext(w,
1492 RING_CTX_TIMESTAMP(engine->mmio_base),
1493 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1494 break;
1495
1496 default:
1497 whitelist_reg_ext(w,
1498 RING_CTX_TIMESTAMP(engine->mmio_base),
1499 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1500 break;
1501 }
1502}
1503
1504static void tgl_whitelist_build(struct intel_engine_cs *engine)
1505{
1506 struct i915_wa_list *w = &engine->whitelist;
1507
1508 switch (engine->class) {
1509 case RENDER_CLASS:
1510 /*
1511 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1512 * Wa_1408556865:tgl
1513 *
1514 * This covers 4 registers which are next to one another :
1515 * - PS_INVOCATION_COUNT
1516 * - PS_INVOCATION_COUNT_UDW
1517 * - PS_DEPTH_COUNT
1518 * - PS_DEPTH_COUNT_UDW
1519 */
1520 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1521 RING_FORCE_TO_NONPRIV_ACCESS_RD |
1522 RING_FORCE_TO_NONPRIV_RANGE_4);
1523
1524 /* Wa_1808121037:tgl */
1525 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1526
1527 /* Wa_1806527549:tgl */
1528 whitelist_reg(w, HIZ_CHICKEN);
1529 break;
1530 default:
1531 whitelist_reg_ext(w,
1532 RING_CTX_TIMESTAMP(engine->mmio_base),
1533 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1534 break;
1535 }
1536}
1537
1538static void dg1_whitelist_build(struct intel_engine_cs *engine)
1539{
1540 struct i915_wa_list *w = &engine->whitelist;
1541
1542 tgl_whitelist_build(engine);
1543
1544 /* GEN:BUG:1409280441:dg1 */
1545 if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) &&
1546 (engine->class == RENDER_CLASS ||
1547 engine->class == COPY_ENGINE_CLASS))
1548 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1549 RING_FORCE_TO_NONPRIV_ACCESS_RD);
1550}
1551
1552void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1553{
1554 struct drm_i915_private *i915 = engine->i915;
1555 struct i915_wa_list *w = &engine->whitelist;
1556
1557 wa_init_start(w, "whitelist", engine->name);
1558
1559 if (IS_DG1(i915))
1560 dg1_whitelist_build(engine);
1561 else if (GRAPHICS_VER(i915) == 12)
1562 tgl_whitelist_build(engine);
1563 else if (GRAPHICS_VER(i915) == 11)
1564 icl_whitelist_build(engine);
1565 else if (IS_CANNONLAKE(i915))
1566 cnl_whitelist_build(engine);
1567 else if (IS_COMETLAKE(i915))
1568 cml_whitelist_build(engine);
1569 else if (IS_COFFEELAKE(i915))
1570 cfl_whitelist_build(engine);
1571 else if (IS_GEMINILAKE(i915))
1572 glk_whitelist_build(engine);
1573 else if (IS_KABYLAKE(i915))
1574 kbl_whitelist_build(engine);
1575 else if (IS_BROXTON(i915))
1576 bxt_whitelist_build(engine);
1577 else if (IS_SKYLAKE(i915))
1578 skl_whitelist_build(engine);
1579 else if (GRAPHICS_VER(i915) <= 8)
1580 ;
1581 else
1582 MISSING_CASE(GRAPHICS_VER(i915));
1583
1584 wa_init_finish(w);
1585}
1586
1587void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1588{
1589 const struct i915_wa_list *wal = &engine->whitelist;
1590 struct intel_uncore *uncore = engine->uncore;
1591 const u32 base = engine->mmio_base;
1592 struct i915_wa *wa;
1593 unsigned int i;
1594
1595 if (!wal->count)
1596 return;
1597
1598 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1599 intel_uncore_write(uncore,
1600 RING_FORCE_TO_NONPRIV(base, i),
1601 i915_mmio_reg_offset(wa->reg));
1602
1603 /* And clear the rest just in case of garbage */
1604 for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1605 intel_uncore_write(uncore,
1606 RING_FORCE_TO_NONPRIV(base, i),
1607 i915_mmio_reg_offset(RING_NOPID(base)));
1608}
1609
1610static void
1611rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1612{
1613 struct drm_i915_private *i915 = engine->i915;
1614
1615 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1616 IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) {
1617 /*
1618 * Wa_1607138336:tgl[a0],dg1[a0]
1619 * Wa_1607063988:tgl[a0],dg1[a0]
1620 */
1621 wa_write_or(wal,
1622 GEN9_CTX_PREEMPT_REG,
1623 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1624 }
1625
1626 if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) {
1627 /*
1628 * Wa_1606679103:tgl
1629 * (see also Wa_1606682166:icl)
1630 */
1631 wa_write_or(wal,
1632 GEN7_SARCHKMD,
1633 GEN7_DISABLE_SAMPLER_PREFETCH);
1634 }
1635
1636 if (IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
1637 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1638 /* Wa_1606931601:tgl,rkl,dg1,adl-s */
1639 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1640
1641 /*
1642 * Wa_1407928979:tgl A*
1643 * Wa_18011464164:tgl[B0+],dg1[B0+]
1644 * Wa_22010931296:tgl[B0+],dg1[B0+]
1645 * Wa_14010919138:rkl,dg1,adl-s
1646 */
1647 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1648 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1649
1650 /*
1651 * Wa_1606700617:tgl,dg1
1652 * Wa_22010271021:tgl,rkl,dg1, adl-s
1653 */
1654 wa_masked_en(wal,
1655 GEN9_CS_DEBUG_MODE1,
1656 FF_DOP_CLOCK_GATE_DISABLE);
1657 }
1658
1659 if (IS_ALDERLAKE_S(i915) || IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1660 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1661 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s */
1662 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1663 GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1664
1665 /*
1666 * Wa_1409085225:tgl
1667 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s
1668 */
1669 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1670 }
1671
1672
1673 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1674 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1675 /*
1676 * Wa_1607030317:tgl
1677 * Wa_1607186500:tgl
1678 * Wa_1607297627:tgl,rkl,dg1[a0]
1679 *
1680 * On TGL and RKL there are multiple entries for this WA in the
1681 * BSpec; some indicate this is an A0-only WA, others indicate
1682 * it applies to all steppings so we trust the "all steppings."
1683 * For DG1 this only applies to A0.
1684 */
1685 wa_masked_en(wal,
1686 GEN6_RC_SLEEP_PSMI_CONTROL,
1687 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1688 GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1689 }
1690
1691 if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1692 /* Wa_1406941453:tgl,rkl,dg1 */
1693 wa_masked_en(wal,
1694 GEN10_SAMPLER_MODE,
1695 ENABLE_SMALLPL);
1696 }
1697
1698 if (GRAPHICS_VER(i915) == 11) {
1699 /* This is not an Wa. Enable for better image quality */
1700 wa_masked_en(wal,
1701 _3D_CHICKEN3,
1702 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1703
1704 /* WaPipelineFlushCoherentLines:icl */
1705 wa_write_or(wal,
1706 GEN8_L3SQCREG4,
1707 GEN8_LQSC_FLUSH_COHERENT_LINES);
1708
1709 /*
1710 * Wa_1405543622:icl
1711 * Formerly known as WaGAPZPriorityScheme
1712 */
1713 wa_write_or(wal,
1714 GEN8_GARBCNTL,
1715 GEN11_ARBITRATION_PRIO_ORDER_MASK);
1716
1717 /*
1718 * Wa_1604223664:icl
1719 * Formerly known as WaL3BankAddressHashing
1720 */
1721 wa_write_clr_set(wal,
1722 GEN8_GARBCNTL,
1723 GEN11_HASH_CTRL_EXCL_MASK,
1724 GEN11_HASH_CTRL_EXCL_BIT0);
1725 wa_write_clr_set(wal,
1726 GEN11_GLBLINVL,
1727 GEN11_BANK_HASH_ADDR_EXCL_MASK,
1728 GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1729
1730 /*
1731 * Wa_1405733216:icl
1732 * Formerly known as WaDisableCleanEvicts
1733 */
1734 wa_write_or(wal,
1735 GEN8_L3SQCREG4,
1736 GEN11_LQSC_CLEAN_EVICT_DISABLE);
1737
1738 /* WaForwardProgressSoftReset:icl */
1739 wa_write_or(wal,
1740 GEN10_SCRATCH_LNCF2,
1741 PMFLUSHDONE_LNICRSDROP |
1742 PMFLUSH_GAPL3UNBLOCK |
1743 PMFLUSHDONE_LNEBLK);
1744
1745 /* Wa_1406609255:icl (pre-prod) */
1746 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1747 wa_write_or(wal,
1748 GEN7_SARCHKMD,
1749 GEN7_DISABLE_DEMAND_PREFETCH);
1750
1751 /* Wa_1606682166:icl */
1752 wa_write_or(wal,
1753 GEN7_SARCHKMD,
1754 GEN7_DISABLE_SAMPLER_PREFETCH);
1755
1756 /* Wa_1409178092:icl */
1757 wa_write_clr_set(wal,
1758 GEN11_SCRATCH2,
1759 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1760 0);
1761
1762 /* WaEnable32PlaneMode:icl */
1763 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1764 GEN11_ENABLE_32_PLANE_MODE);
1765
1766 /*
1767 * Wa_1408615072:icl,ehl (vsunit)
1768 * Wa_1407596294:icl,ehl (hsunit)
1769 */
1770 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1771 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1772
1773 /* Wa_1407352427:icl,ehl */
1774 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1775 PSDUNIT_CLKGATE_DIS);
1776
1777 /* Wa_1406680159:icl,ehl */
1778 wa_write_or(wal,
1779 SUBSLICE_UNIT_LEVEL_CLKGATE,
1780 GWUNIT_CLKGATE_DIS);
1781
1782 /*
1783 * Wa_1408767742:icl[a2..forever],ehl[all]
1784 * Wa_1605460711:icl[a0..c0]
1785 */
1786 wa_write_or(wal,
1787 GEN7_FF_THREAD_MODE,
1788 GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1789
1790 /* Wa_22010271021 */
1791 wa_masked_en(wal,
1792 GEN9_CS_DEBUG_MODE1,
1793 FF_DOP_CLOCK_GATE_DISABLE);
1794 }
1795
1796 if (IS_GRAPHICS_VER(i915, 9, 12)) {
1797 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1798 wa_masked_en(wal,
1799 GEN7_FF_SLICE_CS_CHICKEN1,
1800 GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1801 }
1802
1803 if (IS_SKYLAKE(i915) ||
1804 IS_KABYLAKE(i915) ||
1805 IS_COFFEELAKE(i915) ||
1806 IS_COMETLAKE(i915)) {
1807 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1808 wa_write_or(wal,
1809 GEN8_GARBCNTL,
1810 GEN9_GAPS_TSV_CREDIT_DISABLE);
1811 }
1812
1813 if (IS_BROXTON(i915)) {
1814 /* WaDisablePooledEuLoadBalancingFix:bxt */
1815 wa_masked_en(wal,
1816 FF_SLICE_CS_CHICKEN2,
1817 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1818 }
1819
1820 if (GRAPHICS_VER(i915) == 9) {
1821 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1822 wa_masked_en(wal,
1823 GEN9_CSFE_CHICKEN1_RCS,
1824 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1825
1826 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1827 wa_write_or(wal,
1828 BDW_SCRATCH1,
1829 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1830
1831 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1832 if (IS_GEN9_LP(i915))
1833 wa_write_clr_set(wal,
1834 GEN8_L3SQCREG1,
1835 L3_PRIO_CREDITS_MASK,
1836 L3_GENERAL_PRIO_CREDITS(62) |
1837 L3_HIGH_PRIO_CREDITS(2));
1838
1839 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1840 wa_write_or(wal,
1841 GEN8_L3SQCREG4,
1842 GEN8_LQSC_FLUSH_COHERENT_LINES);
1843
1844 /* Disable atomics in L3 to prevent unrecoverable hangs */
1845 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
1846 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1847 wa_write_clr_set(wal, GEN8_L3SQCREG4,
1848 GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1849 wa_write_clr_set(wal, GEN9_SCRATCH1,
1850 EVICTION_PERF_FIX_ENABLE, 0);
1851 }
1852
1853 if (IS_HASWELL(i915)) {
1854 /* WaSampleCChickenBitEnable:hsw */
1855 wa_masked_en(wal,
1856 HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
1857
1858 wa_masked_dis(wal,
1859 CACHE_MODE_0_GEN7,
1860 /* enable HiZ Raw Stall Optimization */
1861 HIZ_RAW_STALL_OPT_DISABLE);
1862 }
1863
1864 if (IS_VALLEYVIEW(i915)) {
1865 /* WaDisableEarlyCull:vlv */
1866 wa_masked_en(wal,
1867 _3D_CHICKEN3,
1868 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1869
1870 /*
1871 * WaVSThreadDispatchOverride:ivb,vlv
1872 *
1873 * This actually overrides the dispatch
1874 * mode for all thread types.
1875 */
1876 wa_write_clr_set(wal,
1877 GEN7_FF_THREAD_MODE,
1878 GEN7_FF_SCHED_MASK,
1879 GEN7_FF_TS_SCHED_HW |
1880 GEN7_FF_VS_SCHED_HW |
1881 GEN7_FF_DS_SCHED_HW);
1882
1883 /* WaPsdDispatchEnable:vlv */
1884 /* WaDisablePSDDualDispatchEnable:vlv */
1885 wa_masked_en(wal,
1886 GEN7_HALF_SLICE_CHICKEN1,
1887 GEN7_MAX_PS_THREAD_DEP |
1888 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1889 }
1890
1891 if (IS_IVYBRIDGE(i915)) {
1892 /* WaDisableEarlyCull:ivb */
1893 wa_masked_en(wal,
1894 _3D_CHICKEN3,
1895 _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1896
1897 if (0) { /* causes HiZ corruption on ivb:gt1 */
1898 /* enable HiZ Raw Stall Optimization */
1899 wa_masked_dis(wal,
1900 CACHE_MODE_0_GEN7,
1901 HIZ_RAW_STALL_OPT_DISABLE);
1902 }
1903
1904 /*
1905 * WaVSThreadDispatchOverride:ivb,vlv
1906 *
1907 * This actually overrides the dispatch
1908 * mode for all thread types.
1909 */
1910 wa_write_clr_set(wal,
1911 GEN7_FF_THREAD_MODE,
1912 GEN7_FF_SCHED_MASK,
1913 GEN7_FF_TS_SCHED_HW |
1914 GEN7_FF_VS_SCHED_HW |
1915 GEN7_FF_DS_SCHED_HW);
1916
1917 /* WaDisablePSDDualDispatchEnable:ivb */
1918 if (IS_IVB_GT1(i915))
1919 wa_masked_en(wal,
1920 GEN7_HALF_SLICE_CHICKEN1,
1921 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1922 }
1923
1924 if (GRAPHICS_VER(i915) == 7) {
1925 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1926 wa_masked_en(wal,
1927 GFX_MODE_GEN7,
1928 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1929
1930 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
1931 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
1932
1933 /*
1934 * BSpec says this must be set, even though
1935 * WaDisable4x2SubspanOptimization:ivb,hsw
1936 * WaDisable4x2SubspanOptimization isn't listed for VLV.
1937 */
1938 wa_masked_en(wal,
1939 CACHE_MODE_1,
1940 PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
1941
1942 /*
1943 * BSpec recommends 8x4 when MSAA is used,
1944 * however in practice 16x4 seems fastest.
1945 *
1946 * Note that PS/WM thread counts depend on the WIZ hashing
1947 * disable bit, which we don't touch here, but it's good
1948 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
1949 */
1950 wa_add(wal, GEN7_GT_MODE, 0,
1951 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK,
1952 GEN6_WIZ_HASHING_16x4),
1953 GEN6_WIZ_HASHING_16x4);
1954 }
1955
1956 if (IS_GRAPHICS_VER(i915, 6, 7))
1957 /*
1958 * We need to disable the AsyncFlip performance optimisations in
1959 * order to use MI_WAIT_FOR_EVENT within the CS. It should
1960 * already be programmed to '1' on all products.
1961 *
1962 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1963 */
1964 wa_masked_en(wal,
1965 MI_MODE,
1966 ASYNC_FLIP_PERF_DISABLE);
1967
1968 if (GRAPHICS_VER(i915) == 6) {
1969 /*
1970 * Required for the hardware to program scanline values for
1971 * waiting
1972 * WaEnableFlushTlbInvalidationMode:snb
1973 */
1974 wa_masked_en(wal,
1975 GFX_MODE,
1976 GFX_TLB_INVALIDATE_EXPLICIT);
1977
1978 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
1979 wa_masked_en(wal,
1980 _3D_CHICKEN,
1981 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
1982
1983 wa_masked_en(wal,
1984 _3D_CHICKEN3,
1985 /* WaStripsFansDisableFastClipPerformanceFix:snb */
1986 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
1987 /*
1988 * Bspec says:
1989 * "This bit must be set if 3DSTATE_CLIP clip mode is set
1990 * to normal and 3DSTATE_SF number of SF output attributes
1991 * is more than 16."
1992 */
1993 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
1994
1995 /*
1996 * BSpec recommends 8x4 when MSAA is used,
1997 * however in practice 16x4 seems fastest.
1998 *
1999 * Note that PS/WM thread counts depend on the WIZ hashing
2000 * disable bit, which we don't touch here, but it's good
2001 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2002 */
2003 wa_add(wal,
2004 GEN6_GT_MODE, 0,
2005 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
2006 GEN6_WIZ_HASHING_16x4);
2007
2008 /* WaDisable_RenderCache_OperationalFlush:snb */
2009 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2010
2011 /*
2012 * From the Sandybridge PRM, volume 1 part 3, page 24:
2013 * "If this bit is set, STCunit will have LRA as replacement
2014 * policy. [...] This bit must be reset. LRA replacement
2015 * policy is not supported."
2016 */
2017 wa_masked_dis(wal,
2018 CACHE_MODE_0,
2019 CM0_STC_EVICT_DISABLE_LRA_SNB);
2020 }
2021
2022 if (IS_GRAPHICS_VER(i915, 4, 6))
2023 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2024 wa_add(wal, MI_MODE,
2025 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2026 /* XXX bit doesn't stick on Broadwater */
2027 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
2028
2029 if (GRAPHICS_VER(i915) == 4)
2030 /*
2031 * Disable CONSTANT_BUFFER before it is loaded from the context
2032 * image. For as it is loaded, it is executed and the stored
2033 * address may no longer be valid, leading to a GPU hang.
2034 *
2035 * This imposes the requirement that userspace reload their
2036 * CONSTANT_BUFFER on every batch, fortunately a requirement
2037 * they are already accustomed to from before contexts were
2038 * enabled.
2039 */
2040 wa_add(wal, ECOSKPD,
2041 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2042 0 /* XXX bit doesn't stick on Broadwater */);
2043}
2044
2045static void
2046xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2047{
2048 struct drm_i915_private *i915 = engine->i915;
2049
2050 /* WaKBLVECSSemaphoreWaitPoll:kbl */
2051 if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_E0)) {
2052 wa_write(wal,
2053 RING_SEMA_WAIT_POLL(engine->mmio_base),
2054 1);
2055 }
2056}
2057
2058static void
2059engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2060{
2061 if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4))
2062 return;
2063
2064 if (engine->class == RENDER_CLASS)
2065 rcs_engine_wa_init(engine, wal);
2066 else
2067 xcs_engine_wa_init(engine, wal);
2068}
2069
2070void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2071{
2072 struct i915_wa_list *wal = &engine->wa_list;
2073
2074 if (GRAPHICS_VER(engine->i915) < 4)
2075 return;
2076
2077 wa_init_start(wal, "engine", engine->name);
2078 engine_init_workarounds(engine, wal);
2079 wa_init_finish(wal);
2080}
2081
2082void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2083{
2084 wa_list_apply(engine->uncore, &engine->wa_list);
2085}
2086
2087struct mcr_range {
2088 u32 start;
2089 u32 end;
2090};
2091
2092static const struct mcr_range mcr_ranges_gen8[] = {
2093 { .start = 0x5500, .end = 0x55ff },
2094 { .start = 0x7000, .end = 0x7fff },
2095 { .start = 0x9400, .end = 0x97ff },
2096 { .start = 0xb000, .end = 0xb3ff },
2097 { .start = 0xe000, .end = 0xe7ff },
2098 {},
2099};
2100
2101static const struct mcr_range mcr_ranges_gen12[] = {
2102 { .start = 0x8150, .end = 0x815f },
2103 { .start = 0x9520, .end = 0x955f },
2104 { .start = 0xb100, .end = 0xb3ff },
2105 { .start = 0xde80, .end = 0xe8ff },
2106 { .start = 0x24a00, .end = 0x24a7f },
2107 {},
2108};
2109
2110static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2111{
2112 const struct mcr_range *mcr_ranges;
2113 int i;
2114
2115 if (GRAPHICS_VER(i915) >= 12)
2116 mcr_ranges = mcr_ranges_gen12;
2117 else if (GRAPHICS_VER(i915) >= 8)
2118 mcr_ranges = mcr_ranges_gen8;
2119 else
2120 return false;
2121
2122 /*
2123 * Registers in these ranges are affected by the MCR selector
2124 * which only controls CPU initiated MMIO. Routing does not
2125 * work for CS access so we cannot verify them on this path.
2126 */
2127 for (i = 0; mcr_ranges[i].start; i++)
2128 if (offset >= mcr_ranges[i].start &&
2129 offset <= mcr_ranges[i].end)
2130 return true;
2131
2132 return false;
2133}
2134
2135static int
2136wa_list_srm(struct i915_request *rq,
2137 const struct i915_wa_list *wal,
2138 struct i915_vma *vma)
2139{
2140 struct drm_i915_private *i915 = rq->engine->i915;
2141 unsigned int i, count = 0;
2142 const struct i915_wa *wa;
2143 u32 srm, *cs;
2144
2145 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2146 if (GRAPHICS_VER(i915) >= 8)
2147 srm++;
2148
2149 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2150 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2151 count++;
2152 }
2153
2154 cs = intel_ring_begin(rq, 4 * count);
2155 if (IS_ERR(cs))
2156 return PTR_ERR(cs);
2157
2158 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2159 u32 offset = i915_mmio_reg_offset(wa->reg);
2160
2161 if (mcr_range(i915, offset))
2162 continue;
2163
2164 *cs++ = srm;
2165 *cs++ = offset;
2166 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2167 *cs++ = 0;
2168 }
2169 intel_ring_advance(rq, cs);
2170
2171 return 0;
2172}
2173
2174static int engine_wa_list_verify(struct intel_context *ce,
2175 const struct i915_wa_list * const wal,
2176 const char *from)
2177{
2178 const struct i915_wa *wa;
2179 struct i915_request *rq;
2180 struct i915_vma *vma;
2181 struct i915_gem_ww_ctx ww;
2182 unsigned int i;
2183 u32 *results;
2184 int err;
2185
2186 if (!wal->count)
2187 return 0;
2188
2189 vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
2190 wal->count * sizeof(u32));
2191 if (IS_ERR(vma))
2192 return PTR_ERR(vma);
2193
2194 intel_engine_pm_get(ce->engine);
2195 i915_gem_ww_ctx_init(&ww, false);
2196retry:
2197 err = i915_gem_object_lock(vma->obj, &ww);
2198 if (err == 0)
2199 err = intel_context_pin_ww(ce, &ww);
2200 if (err)
2201 goto err_pm;
2202
2203 err = i915_vma_pin_ww(vma, &ww, 0, 0,
2204 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2205 if (err)
2206 goto err_unpin;
2207
2208 rq = i915_request_create(ce);
2209 if (IS_ERR(rq)) {
2210 err = PTR_ERR(rq);
2211 goto err_vma;
2212 }
2213
2214 err = i915_request_await_object(rq, vma->obj, true);
2215 if (err == 0)
2216 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2217 if (err == 0)
2218 err = wa_list_srm(rq, wal, vma);
2219
2220 i915_request_get(rq);
2221 if (err)
2222 i915_request_set_error_once(rq, err);
2223 i915_request_add(rq);
2224
2225 if (err)
2226 goto err_rq;
2227
2228 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2229 err = -ETIME;
2230 goto err_rq;
2231 }
2232
2233 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2234 if (IS_ERR(results)) {
2235 err = PTR_ERR(results);
2236 goto err_rq;
2237 }
2238
2239 err = 0;
2240 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2241 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2242 continue;
2243
2244 if (!wa_verify(wa, results[i], wal->name, from))
2245 err = -ENXIO;
2246 }
2247
2248 i915_gem_object_unpin_map(vma->obj);
2249
2250err_rq:
2251 i915_request_put(rq);
2252err_vma:
2253 i915_vma_unpin(vma);
2254err_unpin:
2255 intel_context_unpin(ce);
2256err_pm:
2257 if (err == -EDEADLK) {
2258 err = i915_gem_ww_ctx_backoff(&ww);
2259 if (!err)
2260 goto retry;
2261 }
2262 i915_gem_ww_ctx_fini(&ww);
2263 intel_engine_pm_put(ce->engine);
2264 i915_vma_put(vma);
2265 return err;
2266}
2267
2268int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2269 const char *from)
2270{
2271 return engine_wa_list_verify(engine->kernel_context,
2272 &engine->wa_list,
2273 from);
2274}
2275
2276#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2277#include "selftest_workarounds.c"
2278#endif