Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include "display/intel_de.h"
29#include "display/intel_display_trace.h"
30#include "display/skl_watermark.h"
31
32#include "gt/intel_engine_regs.h"
33#include "gt/intel_gt.h"
34#include "gt/intel_gt_mcr.h"
35#include "gt/intel_gt_regs.h"
36
37#include "i915_drv.h"
38#include "intel_mchbar_regs.h"
39#include "intel_pm.h"
40#include "vlv_sideband.h"
41
42struct drm_i915_clock_gating_funcs {
43 void (*init_clock_gating)(struct drm_i915_private *i915);
44};
45
46/* used in computing the new watermarks state */
47struct intel_wm_config {
48 unsigned int num_pipes_active;
49 bool sprites_enabled;
50 bool sprites_scaled;
51};
52
53static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
54{
55 if (HAS_LLC(dev_priv)) {
56 /*
57 * WaCompressedResourceDisplayNewHashMode:skl,kbl
58 * Display WA #0390: skl,kbl
59 *
60 * Must match Sampler, Pixel Back End, and Media. See
61 * WaCompressedResourceSamplerPbeMediaNewHashMode.
62 */
63 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
64 }
65
66 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
67 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
68
69 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
70 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
71
72 /*
73 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
74 * Display WA #0859: skl,bxt,kbl,glk,cfl
75 */
76 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
77}
78
79static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
80{
81 gen9_init_clock_gating(dev_priv);
82
83 /* WaDisableSDEUnitClockGating:bxt */
84 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
85
86 /*
87 * FIXME:
88 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
89 */
90 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
91
92 /*
93 * Wa: Backlight PWM may stop in the asserted state, causing backlight
94 * to stay fully on.
95 */
96 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
97 PWM1_GATING_DIS | PWM2_GATING_DIS);
98
99 /*
100 * Lower the display internal timeout.
101 * This is needed to avoid any hard hangs when DSI port PLL
102 * is off and a MMIO access is attempted by any privilege
103 * application, using batch buffers or any other means.
104 */
105 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
106
107 /*
108 * WaFbcTurnOffFbcWatermark:bxt
109 * Display WA #0562: bxt
110 */
111 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
112
113 /*
114 * WaFbcHighMemBwCorruptionAvoidance:bxt
115 * Display WA #0883: bxt
116 */
117 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
118}
119
120static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124 /*
125 * WaDisablePWMClockGating:glk
126 * Backlight PWM may stop in the asserted state, causing backlight
127 * to stay fully on.
128 */
129 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
130 PWM1_GATING_DIS | PWM2_GATING_DIS);
131}
132
133static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
134{
135 u32 tmp;
136
137 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
138
139 switch (tmp & CLKCFG_FSB_MASK) {
140 case CLKCFG_FSB_533:
141 dev_priv->fsb_freq = 533; /* 133*4 */
142 break;
143 case CLKCFG_FSB_800:
144 dev_priv->fsb_freq = 800; /* 200*4 */
145 break;
146 case CLKCFG_FSB_667:
147 dev_priv->fsb_freq = 667; /* 167*4 */
148 break;
149 case CLKCFG_FSB_400:
150 dev_priv->fsb_freq = 400; /* 100*4 */
151 break;
152 }
153
154 switch (tmp & CLKCFG_MEM_MASK) {
155 case CLKCFG_MEM_533:
156 dev_priv->mem_freq = 533;
157 break;
158 case CLKCFG_MEM_667:
159 dev_priv->mem_freq = 667;
160 break;
161 case CLKCFG_MEM_800:
162 dev_priv->mem_freq = 800;
163 break;
164 }
165
166 /* detect pineview DDR3 setting */
167 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
168 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
169}
170
171static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
172{
173 u16 ddrpll, csipll;
174
175 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
176 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
177
178 switch (ddrpll & 0xff) {
179 case 0xc:
180 dev_priv->mem_freq = 800;
181 break;
182 case 0x10:
183 dev_priv->mem_freq = 1066;
184 break;
185 case 0x14:
186 dev_priv->mem_freq = 1333;
187 break;
188 case 0x18:
189 dev_priv->mem_freq = 1600;
190 break;
191 default:
192 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
193 ddrpll & 0xff);
194 dev_priv->mem_freq = 0;
195 break;
196 }
197
198 switch (csipll & 0x3ff) {
199 case 0x00c:
200 dev_priv->fsb_freq = 3200;
201 break;
202 case 0x00e:
203 dev_priv->fsb_freq = 3733;
204 break;
205 case 0x010:
206 dev_priv->fsb_freq = 4266;
207 break;
208 case 0x012:
209 dev_priv->fsb_freq = 4800;
210 break;
211 case 0x014:
212 dev_priv->fsb_freq = 5333;
213 break;
214 case 0x016:
215 dev_priv->fsb_freq = 5866;
216 break;
217 case 0x018:
218 dev_priv->fsb_freq = 6400;
219 break;
220 default:
221 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
222 csipll & 0x3ff);
223 dev_priv->fsb_freq = 0;
224 break;
225 }
226}
227
228static const struct cxsr_latency cxsr_latency_table[] = {
229 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
230 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
231 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
232 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
233 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
234
235 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
236 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
237 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
238 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
239 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
240
241 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
242 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
243 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
244 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
245 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
246
247 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
248 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
249 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
250 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
251 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
252
253 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
254 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
255 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
256 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
257 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
258
259 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
260 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
261 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
262 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
263 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
264};
265
266static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
267 bool is_ddr3,
268 int fsb,
269 int mem)
270{
271 const struct cxsr_latency *latency;
272 int i;
273
274 if (fsb == 0 || mem == 0)
275 return NULL;
276
277 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
278 latency = &cxsr_latency_table[i];
279 if (is_desktop == latency->is_desktop &&
280 is_ddr3 == latency->is_ddr3 &&
281 fsb == latency->fsb_freq && mem == latency->mem_freq)
282 return latency;
283 }
284
285 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
286
287 return NULL;
288}
289
290static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
291{
292 u32 val;
293
294 vlv_punit_get(dev_priv);
295
296 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
297 if (enable)
298 val &= ~FORCE_DDR_HIGH_FREQ;
299 else
300 val |= FORCE_DDR_HIGH_FREQ;
301 val &= ~FORCE_DDR_LOW_FREQ;
302 val |= FORCE_DDR_FREQ_REQ_ACK;
303 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
304
305 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
306 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
307 drm_err(&dev_priv->drm,
308 "timed out waiting for Punit DDR DVFS request\n");
309
310 vlv_punit_put(dev_priv);
311}
312
313static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
314{
315 u32 val;
316
317 vlv_punit_get(dev_priv);
318
319 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
320 if (enable)
321 val |= DSP_MAXFIFO_PM5_ENABLE;
322 else
323 val &= ~DSP_MAXFIFO_PM5_ENABLE;
324 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
325
326 vlv_punit_put(dev_priv);
327}
328
329#define FW_WM(value, plane) \
330 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
331
332static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
333{
334 bool was_enabled;
335 u32 val;
336
337 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
338 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
339 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
340 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
341 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
342 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
343 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
344 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
345 } else if (IS_PINEVIEW(dev_priv)) {
346 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
347 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
348 if (enable)
349 val |= PINEVIEW_SELF_REFRESH_EN;
350 else
351 val &= ~PINEVIEW_SELF_REFRESH_EN;
352 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
353 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
354 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
355 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
356 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
357 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
358 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
359 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
360 } else if (IS_I915GM(dev_priv)) {
361 /*
362 * FIXME can't find a bit like this for 915G, and
363 * and yet it does have the related watermark in
364 * FW_BLC_SELF. What's going on?
365 */
366 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
367 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
368 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
369 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
370 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
371 } else {
372 return false;
373 }
374
375 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
376
377 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
378 str_enabled_disabled(enable),
379 str_enabled_disabled(was_enabled));
380
381 return was_enabled;
382}
383
384/**
385 * intel_set_memory_cxsr - Configure CxSR state
386 * @dev_priv: i915 device
387 * @enable: Allow vs. disallow CxSR
388 *
389 * Allow or disallow the system to enter a special CxSR
390 * (C-state self refresh) state. What typically happens in CxSR mode
391 * is that several display FIFOs may get combined into a single larger
392 * FIFO for a particular plane (so called max FIFO mode) to allow the
393 * system to defer memory fetches longer, and the memory will enter
394 * self refresh.
395 *
396 * Note that enabling CxSR does not guarantee that the system enter
397 * this special mode, nor does it guarantee that the system stays
398 * in that mode once entered. So this just allows/disallows the system
399 * to autonomously utilize the CxSR mode. Other factors such as core
400 * C-states will affect when/if the system actually enters/exits the
401 * CxSR mode.
402 *
403 * Note that on VLV/CHV this actually only controls the max FIFO mode,
404 * and the system is free to enter/exit memory self refresh at any time
405 * even when the use of CxSR has been disallowed.
406 *
407 * While the system is actually in the CxSR/max FIFO mode, some plane
408 * control registers will not get latched on vblank. Thus in order to
409 * guarantee the system will respond to changes in the plane registers
410 * we must always disallow CxSR prior to making changes to those registers.
411 * Unfortunately the system will re-evaluate the CxSR conditions at
412 * frame start which happens after vblank start (which is when the plane
413 * registers would get latched), so we can't proceed with the plane update
414 * during the same frame where we disallowed CxSR.
415 *
416 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
417 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
418 * the hardware w.r.t. HPLL SR when writing to plane registers.
419 * Disallowing just CxSR is sufficient.
420 */
421bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
422{
423 bool ret;
424
425 mutex_lock(&dev_priv->display.wm.wm_mutex);
426 ret = _intel_set_memory_cxsr(dev_priv, enable);
427 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
428 dev_priv->display.wm.vlv.cxsr = enable;
429 else if (IS_G4X(dev_priv))
430 dev_priv->display.wm.g4x.cxsr = enable;
431 mutex_unlock(&dev_priv->display.wm.wm_mutex);
432
433 return ret;
434}
435
436/*
437 * Latency for FIFO fetches is dependent on several factors:
438 * - memory configuration (speed, channels)
439 * - chipset
440 * - current MCH state
441 * It can be fairly high in some situations, so here we assume a fairly
442 * pessimal value. It's a tradeoff between extra memory fetches (if we
443 * set this value too high, the FIFO will fetch frequently to stay full)
444 * and power consumption (set it too low to save power and we might see
445 * FIFO underruns and display "flicker").
446 *
447 * A value of 5us seems to be a good balance; safe for very low end
448 * platforms but not overly aggressive on lower latency configs.
449 */
450static const int pessimal_latency_ns = 5000;
451
452#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
453 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
454
455static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
456{
457 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
459 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
460 enum pipe pipe = crtc->pipe;
461 int sprite0_start, sprite1_start;
462 u32 dsparb, dsparb2, dsparb3;
463
464 switch (pipe) {
465 case PIPE_A:
466 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
467 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
468 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
469 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
470 break;
471 case PIPE_B:
472 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
473 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
474 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
475 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
476 break;
477 case PIPE_C:
478 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
479 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
480 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
481 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
482 break;
483 default:
484 MISSING_CASE(pipe);
485 return;
486 }
487
488 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
489 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
490 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
491 fifo_state->plane[PLANE_CURSOR] = 63;
492}
493
494static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
495 enum i9xx_plane_id i9xx_plane)
496{
497 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
498 int size;
499
500 size = dsparb & 0x7f;
501 if (i9xx_plane == PLANE_B)
502 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
503
504 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
505 dsparb, plane_name(i9xx_plane), size);
506
507 return size;
508}
509
510static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
511 enum i9xx_plane_id i9xx_plane)
512{
513 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
514 int size;
515
516 size = dsparb & 0x1ff;
517 if (i9xx_plane == PLANE_B)
518 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
519 size >>= 1; /* Convert to cachelines */
520
521 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
522 dsparb, plane_name(i9xx_plane), size);
523
524 return size;
525}
526
527static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
528 enum i9xx_plane_id i9xx_plane)
529{
530 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
531 int size;
532
533 size = dsparb & 0x7f;
534 size >>= 2; /* Convert to cachelines */
535
536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
538
539 return size;
540}
541
542/* Pineview has different values for various configs */
543static const struct intel_watermark_params pnv_display_wm = {
544 .fifo_size = PINEVIEW_DISPLAY_FIFO,
545 .max_wm = PINEVIEW_MAX_WM,
546 .default_wm = PINEVIEW_DFT_WM,
547 .guard_size = PINEVIEW_GUARD_WM,
548 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
549};
550
551static const struct intel_watermark_params pnv_display_hplloff_wm = {
552 .fifo_size = PINEVIEW_DISPLAY_FIFO,
553 .max_wm = PINEVIEW_MAX_WM,
554 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
555 .guard_size = PINEVIEW_GUARD_WM,
556 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
557};
558
559static const struct intel_watermark_params pnv_cursor_wm = {
560 .fifo_size = PINEVIEW_CURSOR_FIFO,
561 .max_wm = PINEVIEW_CURSOR_MAX_WM,
562 .default_wm = PINEVIEW_CURSOR_DFT_WM,
563 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
564 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
565};
566
567static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
568 .fifo_size = PINEVIEW_CURSOR_FIFO,
569 .max_wm = PINEVIEW_CURSOR_MAX_WM,
570 .default_wm = PINEVIEW_CURSOR_DFT_WM,
571 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
572 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
573};
574
575static const struct intel_watermark_params i965_cursor_wm_info = {
576 .fifo_size = I965_CURSOR_FIFO,
577 .max_wm = I965_CURSOR_MAX_WM,
578 .default_wm = I965_CURSOR_DFT_WM,
579 .guard_size = 2,
580 .cacheline_size = I915_FIFO_LINE_SIZE,
581};
582
583static const struct intel_watermark_params i945_wm_info = {
584 .fifo_size = I945_FIFO_SIZE,
585 .max_wm = I915_MAX_WM,
586 .default_wm = 1,
587 .guard_size = 2,
588 .cacheline_size = I915_FIFO_LINE_SIZE,
589};
590
591static const struct intel_watermark_params i915_wm_info = {
592 .fifo_size = I915_FIFO_SIZE,
593 .max_wm = I915_MAX_WM,
594 .default_wm = 1,
595 .guard_size = 2,
596 .cacheline_size = I915_FIFO_LINE_SIZE,
597};
598
599static const struct intel_watermark_params i830_a_wm_info = {
600 .fifo_size = I855GM_FIFO_SIZE,
601 .max_wm = I915_MAX_WM,
602 .default_wm = 1,
603 .guard_size = 2,
604 .cacheline_size = I830_FIFO_LINE_SIZE,
605};
606
607static const struct intel_watermark_params i830_bc_wm_info = {
608 .fifo_size = I855GM_FIFO_SIZE,
609 .max_wm = I915_MAX_WM/2,
610 .default_wm = 1,
611 .guard_size = 2,
612 .cacheline_size = I830_FIFO_LINE_SIZE,
613};
614
615static const struct intel_watermark_params i845_wm_info = {
616 .fifo_size = I830_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
618 .default_wm = 1,
619 .guard_size = 2,
620 .cacheline_size = I830_FIFO_LINE_SIZE,
621};
622
623/**
624 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
625 * @pixel_rate: Pipe pixel rate in kHz
626 * @cpp: Plane bytes per pixel
627 * @latency: Memory wakeup latency in 0.1us units
628 *
629 * Compute the watermark using the method 1 or "small buffer"
630 * formula. The caller may additonally add extra cachelines
631 * to account for TLB misses and clock crossings.
632 *
633 * This method is concerned with the short term drain rate
634 * of the FIFO, ie. it does not account for blanking periods
635 * which would effectively reduce the average drain rate across
636 * a longer period. The name "small" refers to the fact the
637 * FIFO is relatively small compared to the amount of data
638 * fetched.
639 *
640 * The FIFO level vs. time graph might look something like:
641 *
642 * |\ |\
643 * | \ | \
644 * __---__---__ (- plane active, _ blanking)
645 * -> time
646 *
647 * or perhaps like this:
648 *
649 * |\|\ |\|\
650 * __----__----__ (- plane active, _ blanking)
651 * -> time
652 *
653 * Returns:
654 * The watermark in bytes
655 */
656static unsigned int intel_wm_method1(unsigned int pixel_rate,
657 unsigned int cpp,
658 unsigned int latency)
659{
660 u64 ret;
661
662 ret = mul_u32_u32(pixel_rate, cpp * latency);
663 ret = DIV_ROUND_UP_ULL(ret, 10000);
664
665 return ret;
666}
667
668/**
669 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @htotal: Pipe horizontal total
672 * @width: Plane width in pixels
673 * @cpp: Plane bytes per pixel
674 * @latency: Memory wakeup latency in 0.1us units
675 *
676 * Compute the watermark using the method 2 or "large buffer"
677 * formula. The caller may additonally add extra cachelines
678 * to account for TLB misses and clock crossings.
679 *
680 * This method is concerned with the long term drain rate
681 * of the FIFO, ie. it does account for blanking periods
682 * which effectively reduce the average drain rate across
683 * a longer period. The name "large" refers to the fact the
684 * FIFO is relatively large compared to the amount of data
685 * fetched.
686 *
687 * The FIFO level vs. time graph might look something like:
688 *
689 * |\___ |\___
690 * | \___ | \___
691 * | \ | \
692 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
693 * -> time
694 *
695 * Returns:
696 * The watermark in bytes
697 */
698static unsigned int intel_wm_method2(unsigned int pixel_rate,
699 unsigned int htotal,
700 unsigned int width,
701 unsigned int cpp,
702 unsigned int latency)
703{
704 unsigned int ret;
705
706 /*
707 * FIXME remove once all users are computing
708 * watermarks in the correct place.
709 */
710 if (WARN_ON_ONCE(htotal == 0))
711 htotal = 1;
712
713 ret = (latency * pixel_rate) / (htotal * 10000);
714 ret = (ret + 1) * width * cpp;
715
716 return ret;
717}
718
719/**
720 * intel_calculate_wm - calculate watermark level
721 * @pixel_rate: pixel clock
722 * @wm: chip FIFO params
723 * @fifo_size: size of the FIFO buffer
724 * @cpp: bytes per pixel
725 * @latency_ns: memory latency for the platform
726 *
727 * Calculate the watermark level (the level at which the display plane will
728 * start fetching from memory again). Each chip has a different display
729 * FIFO size and allocation, so the caller needs to figure that out and pass
730 * in the correct intel_watermark_params structure.
731 *
732 * As the pixel clock runs, the FIFO will be drained at a rate that depends
733 * on the pixel size. When it reaches the watermark level, it'll start
734 * fetching FIFO line sized based chunks from memory until the FIFO fills
735 * past the watermark point. If the FIFO drains completely, a FIFO underrun
736 * will occur, and a display engine hang could result.
737 */
738static unsigned int intel_calculate_wm(int pixel_rate,
739 const struct intel_watermark_params *wm,
740 int fifo_size, int cpp,
741 unsigned int latency_ns)
742{
743 int entries, wm_size;
744
745 /*
746 * Note: we need to make sure we don't overflow for various clock &
747 * latency values.
748 * clocks go from a few thousand to several hundred thousand.
749 * latency is usually a few thousand
750 */
751 entries = intel_wm_method1(pixel_rate, cpp,
752 latency_ns / 100);
753 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
754 wm->guard_size;
755 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
756
757 wm_size = fifo_size - entries;
758 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
759
760 /* Don't promote wm_size to unsigned... */
761 if (wm_size > wm->max_wm)
762 wm_size = wm->max_wm;
763 if (wm_size <= 0)
764 wm_size = wm->default_wm;
765
766 /*
767 * Bspec seems to indicate that the value shouldn't be lower than
768 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
769 * Lets go for 8 which is the burst size since certain platforms
770 * already use a hardcoded 8 (which is what the spec says should be
771 * done).
772 */
773 if (wm_size <= 8)
774 wm_size = 8;
775
776 return wm_size;
777}
778
779static bool is_disabling(int old, int new, int threshold)
780{
781 return old >= threshold && new < threshold;
782}
783
784static bool is_enabling(int old, int new, int threshold)
785{
786 return old < threshold && new >= threshold;
787}
788
789static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
790{
791 return dev_priv->display.wm.max_level + 1;
792}
793
794bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
795 const struct intel_plane_state *plane_state)
796{
797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
798
799 /* FIXME check the 'enable' instead */
800 if (!crtc_state->hw.active)
801 return false;
802
803 /*
804 * Treat cursor with fb as always visible since cursor updates
805 * can happen faster than the vrefresh rate, and the current
806 * watermark code doesn't handle that correctly. Cursor updates
807 * which set/clear the fb or change the cursor size are going
808 * to get throttled by intel_legacy_cursor_update() to work
809 * around this problem with the watermark code.
810 */
811 if (plane->id == PLANE_CURSOR)
812 return plane_state->hw.fb != NULL;
813 else
814 return plane_state->uapi.visible;
815}
816
817static bool intel_crtc_active(struct intel_crtc *crtc)
818{
819 /* Be paranoid as we can arrive here with only partial
820 * state retrieved from the hardware during setup.
821 *
822 * We can ditch the adjusted_mode.crtc_clock check as soon
823 * as Haswell has gained clock readout/fastboot support.
824 *
825 * We can ditch the crtc->primary->state->fb check as soon as we can
826 * properly reconstruct framebuffers.
827 *
828 * FIXME: The intel_crtc->active here should be switched to
829 * crtc->state->active once we have proper CRTC states wired up
830 * for atomic.
831 */
832 return crtc && crtc->active && crtc->base.primary->state->fb &&
833 crtc->config->hw.adjusted_mode.crtc_clock;
834}
835
836static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
837{
838 struct intel_crtc *crtc, *enabled = NULL;
839
840 for_each_intel_crtc(&dev_priv->drm, crtc) {
841 if (intel_crtc_active(crtc)) {
842 if (enabled)
843 return NULL;
844 enabled = crtc;
845 }
846 }
847
848 return enabled;
849}
850
851static void pnv_update_wm(struct drm_i915_private *dev_priv)
852{
853 struct intel_crtc *crtc;
854 const struct cxsr_latency *latency;
855 u32 reg;
856 unsigned int wm;
857
858 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
859 dev_priv->is_ddr3,
860 dev_priv->fsb_freq,
861 dev_priv->mem_freq);
862 if (!latency) {
863 drm_dbg_kms(&dev_priv->drm,
864 "Unknown FSB/MEM found, disable CxSR\n");
865 intel_set_memory_cxsr(dev_priv, false);
866 return;
867 }
868
869 crtc = single_enabled_crtc(dev_priv);
870 if (crtc) {
871 const struct drm_framebuffer *fb =
872 crtc->base.primary->state->fb;
873 int pixel_rate = crtc->config->pixel_rate;
874 int cpp = fb->format->cpp[0];
875
876 /* Display SR */
877 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
878 pnv_display_wm.fifo_size,
879 cpp, latency->display_sr);
880 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
881 reg &= ~DSPFW_SR_MASK;
882 reg |= FW_WM(wm, SR);
883 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
884 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
885
886 /* cursor SR */
887 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
888 pnv_display_wm.fifo_size,
889 4, latency->cursor_sr);
890 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
891 FW_WM(wm, CURSOR_SR));
892
893 /* Display HPLL off SR */
894 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
895 pnv_display_hplloff_wm.fifo_size,
896 cpp, latency->display_hpll_disable);
897 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
898
899 /* cursor HPLL off SR */
900 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
901 pnv_display_hplloff_wm.fifo_size,
902 4, latency->cursor_hpll_disable);
903 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
904 reg &= ~DSPFW_HPLL_CURSOR_MASK;
905 reg |= FW_WM(wm, HPLL_CURSOR);
906 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
907 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
908
909 intel_set_memory_cxsr(dev_priv, true);
910 } else {
911 intel_set_memory_cxsr(dev_priv, false);
912 }
913}
914
915/*
916 * Documentation says:
917 * "If the line size is small, the TLB fetches can get in the way of the
918 * data fetches, causing some lag in the pixel data return which is not
919 * accounted for in the above formulas. The following adjustment only
920 * needs to be applied if eight whole lines fit in the buffer at once.
921 * The WM is adjusted upwards by the difference between the FIFO size
922 * and the size of 8 whole lines. This adjustment is always performed
923 * in the actual pixel depth regardless of whether FBC is enabled or not."
924 */
925static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
926{
927 int tlb_miss = fifo_size * 64 - width * cpp * 8;
928
929 return max(0, tlb_miss);
930}
931
932static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
933 const struct g4x_wm_values *wm)
934{
935 enum pipe pipe;
936
937 for_each_pipe(dev_priv, pipe)
938 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
939
940 intel_uncore_write(&dev_priv->uncore, DSPFW1,
941 FW_WM(wm->sr.plane, SR) |
942 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
943 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
944 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
945 intel_uncore_write(&dev_priv->uncore, DSPFW2,
946 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
947 FW_WM(wm->sr.fbc, FBC_SR) |
948 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
949 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
950 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
951 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
952 intel_uncore_write(&dev_priv->uncore, DSPFW3,
953 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
954 FW_WM(wm->sr.cursor, CURSOR_SR) |
955 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
956 FW_WM(wm->hpll.plane, HPLL_SR));
957
958 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
959}
960
961#define FW_WM_VLV(value, plane) \
962 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
963
964static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
965 const struct vlv_wm_values *wm)
966{
967 enum pipe pipe;
968
969 for_each_pipe(dev_priv, pipe) {
970 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
971
972 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
973 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
974 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
975 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
976 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
977 }
978
979 /*
980 * Zero the (unused) WM1 watermarks, and also clear all the
981 * high order bits so that there are no out of bounds values
982 * present in the registers during the reprogramming.
983 */
984 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
985 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
986 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
987 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
988 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
989
990 intel_uncore_write(&dev_priv->uncore, DSPFW1,
991 FW_WM(wm->sr.plane, SR) |
992 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
993 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
994 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
995 intel_uncore_write(&dev_priv->uncore, DSPFW2,
996 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
998 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
999 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1000 FW_WM(wm->sr.cursor, CURSOR_SR));
1001
1002 if (IS_CHERRYVIEW(dev_priv)) {
1003 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1004 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1005 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1006 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1007 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1008 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1009 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1010 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1011 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1012 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1013 FW_WM(wm->sr.plane >> 9, SR_HI) |
1014 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1015 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1016 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1017 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1018 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1019 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1020 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1021 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1022 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1023 } else {
1024 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1025 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1026 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1027 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1028 FW_WM(wm->sr.plane >> 9, SR_HI) |
1029 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1030 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1031 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1032 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1033 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1034 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1035 }
1036
1037 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1038}
1039
1040#undef FW_WM_VLV
1041
1042static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1043{
1044 /* all latencies in usec */
1045 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1046 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1047 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1048
1049 dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1050}
1051
1052static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1053{
1054 /*
1055 * DSPCNTR[13] supposedly controls whether the
1056 * primary plane can use the FIFO space otherwise
1057 * reserved for the sprite plane. It's not 100% clear
1058 * what the actual FIFO size is, but it looks like we
1059 * can happily set both primary and sprite watermarks
1060 * up to 127 cachelines. So that would seem to mean
1061 * that either DSPCNTR[13] doesn't do anything, or that
1062 * the total FIFO is >= 256 cachelines in size. Either
1063 * way, we don't seem to have to worry about this
1064 * repartitioning as the maximum watermark value the
1065 * register can hold for each plane is lower than the
1066 * minimum FIFO size.
1067 */
1068 switch (plane_id) {
1069 case PLANE_CURSOR:
1070 return 63;
1071 case PLANE_PRIMARY:
1072 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1073 case PLANE_SPRITE0:
1074 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1075 default:
1076 MISSING_CASE(plane_id);
1077 return 0;
1078 }
1079}
1080
1081static int g4x_fbc_fifo_size(int level)
1082{
1083 switch (level) {
1084 case G4X_WM_LEVEL_SR:
1085 return 7;
1086 case G4X_WM_LEVEL_HPLL:
1087 return 15;
1088 default:
1089 MISSING_CASE(level);
1090 return 0;
1091 }
1092}
1093
1094static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1095 const struct intel_plane_state *plane_state,
1096 int level)
1097{
1098 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1099 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1100 const struct drm_display_mode *pipe_mode =
1101 &crtc_state->hw.pipe_mode;
1102 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1103 unsigned int pixel_rate, htotal, cpp, width, wm;
1104
1105 if (latency == 0)
1106 return USHRT_MAX;
1107
1108 if (!intel_wm_plane_visible(crtc_state, plane_state))
1109 return 0;
1110
1111 cpp = plane_state->hw.fb->format->cpp[0];
1112
1113 /*
1114 * WaUse32BppForSRWM:ctg,elk
1115 *
1116 * The spec fails to list this restriction for the
1117 * HPLL watermark, which seems a little strange.
1118 * Let's use 32bpp for the HPLL watermark as well.
1119 */
1120 if (plane->id == PLANE_PRIMARY &&
1121 level != G4X_WM_LEVEL_NORMAL)
1122 cpp = max(cpp, 4u);
1123
1124 pixel_rate = crtc_state->pixel_rate;
1125 htotal = pipe_mode->crtc_htotal;
1126 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1127
1128 if (plane->id == PLANE_CURSOR) {
1129 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1130 } else if (plane->id == PLANE_PRIMARY &&
1131 level == G4X_WM_LEVEL_NORMAL) {
1132 wm = intel_wm_method1(pixel_rate, cpp, latency);
1133 } else {
1134 unsigned int small, large;
1135
1136 small = intel_wm_method1(pixel_rate, cpp, latency);
1137 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1138
1139 wm = min(small, large);
1140 }
1141
1142 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1143 width, cpp);
1144
1145 wm = DIV_ROUND_UP(wm, 64) + 2;
1146
1147 return min_t(unsigned int, wm, USHRT_MAX);
1148}
1149
1150static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1151 int level, enum plane_id plane_id, u16 value)
1152{
1153 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1154 bool dirty = false;
1155
1156 for (; level < intel_wm_num_levels(dev_priv); level++) {
1157 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1158
1159 dirty |= raw->plane[plane_id] != value;
1160 raw->plane[plane_id] = value;
1161 }
1162
1163 return dirty;
1164}
1165
1166static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1167 int level, u16 value)
1168{
1169 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1170 bool dirty = false;
1171
1172 /* NORMAL level doesn't have an FBC watermark */
1173 level = max(level, G4X_WM_LEVEL_SR);
1174
1175 for (; level < intel_wm_num_levels(dev_priv); level++) {
1176 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1177
1178 dirty |= raw->fbc != value;
1179 raw->fbc = value;
1180 }
1181
1182 return dirty;
1183}
1184
1185static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1186 const struct intel_plane_state *plane_state,
1187 u32 pri_val);
1188
1189static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1190 const struct intel_plane_state *plane_state)
1191{
1192 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1193 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1195 enum plane_id plane_id = plane->id;
1196 bool dirty = false;
1197 int level;
1198
1199 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1200 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1201 if (plane_id == PLANE_PRIMARY)
1202 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1203 goto out;
1204 }
1205
1206 for (level = 0; level < num_levels; level++) {
1207 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1208 int wm, max_wm;
1209
1210 wm = g4x_compute_wm(crtc_state, plane_state, level);
1211 max_wm = g4x_plane_fifo_size(plane_id, level);
1212
1213 if (wm > max_wm)
1214 break;
1215
1216 dirty |= raw->plane[plane_id] != wm;
1217 raw->plane[plane_id] = wm;
1218
1219 if (plane_id != PLANE_PRIMARY ||
1220 level == G4X_WM_LEVEL_NORMAL)
1221 continue;
1222
1223 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1224 raw->plane[plane_id]);
1225 max_wm = g4x_fbc_fifo_size(level);
1226
1227 /*
1228 * FBC wm is not mandatory as we
1229 * can always just disable its use.
1230 */
1231 if (wm > max_wm)
1232 wm = USHRT_MAX;
1233
1234 dirty |= raw->fbc != wm;
1235 raw->fbc = wm;
1236 }
1237
1238 /* mark watermarks as invalid */
1239 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1240
1241 if (plane_id == PLANE_PRIMARY)
1242 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1243
1244 out:
1245 if (dirty) {
1246 drm_dbg_kms(&dev_priv->drm,
1247 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1248 plane->base.name,
1249 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1250 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1251 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1252
1253 if (plane_id == PLANE_PRIMARY)
1254 drm_dbg_kms(&dev_priv->drm,
1255 "FBC watermarks: SR=%d, HPLL=%d\n",
1256 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1257 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1258 }
1259
1260 return dirty;
1261}
1262
1263static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1264 enum plane_id plane_id, int level)
1265{
1266 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1267
1268 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1269}
1270
1271static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1272 int level)
1273{
1274 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1275
1276 if (level > dev_priv->display.wm.max_level)
1277 return false;
1278
1279 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1280 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1281 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1282}
1283
1284/* mark all levels starting from 'level' as invalid */
1285static void g4x_invalidate_wms(struct intel_crtc *crtc,
1286 struct g4x_wm_state *wm_state, int level)
1287{
1288 if (level <= G4X_WM_LEVEL_NORMAL) {
1289 enum plane_id plane_id;
1290
1291 for_each_plane_id_on_crtc(crtc, plane_id)
1292 wm_state->wm.plane[plane_id] = USHRT_MAX;
1293 }
1294
1295 if (level <= G4X_WM_LEVEL_SR) {
1296 wm_state->cxsr = false;
1297 wm_state->sr.cursor = USHRT_MAX;
1298 wm_state->sr.plane = USHRT_MAX;
1299 wm_state->sr.fbc = USHRT_MAX;
1300 }
1301
1302 if (level <= G4X_WM_LEVEL_HPLL) {
1303 wm_state->hpll_en = false;
1304 wm_state->hpll.cursor = USHRT_MAX;
1305 wm_state->hpll.plane = USHRT_MAX;
1306 wm_state->hpll.fbc = USHRT_MAX;
1307 }
1308}
1309
1310static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1311 int level)
1312{
1313 if (level < G4X_WM_LEVEL_SR)
1314 return false;
1315
1316 if (level >= G4X_WM_LEVEL_SR &&
1317 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1318 return false;
1319
1320 if (level >= G4X_WM_LEVEL_HPLL &&
1321 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1322 return false;
1323
1324 return true;
1325}
1326
1327static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1328{
1329 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1330 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1331 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1332 const struct g4x_pipe_wm *raw;
1333 enum plane_id plane_id;
1334 int level;
1335
1336 level = G4X_WM_LEVEL_NORMAL;
1337 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1338 goto out;
1339
1340 raw = &crtc_state->wm.g4x.raw[level];
1341 for_each_plane_id_on_crtc(crtc, plane_id)
1342 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1343
1344 level = G4X_WM_LEVEL_SR;
1345 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1346 goto out;
1347
1348 raw = &crtc_state->wm.g4x.raw[level];
1349 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1350 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1351 wm_state->sr.fbc = raw->fbc;
1352
1353 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1354
1355 level = G4X_WM_LEVEL_HPLL;
1356 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1357 goto out;
1358
1359 raw = &crtc_state->wm.g4x.raw[level];
1360 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1361 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1362 wm_state->hpll.fbc = raw->fbc;
1363
1364 wm_state->hpll_en = wm_state->cxsr;
1365
1366 level++;
1367
1368 out:
1369 if (level == G4X_WM_LEVEL_NORMAL)
1370 return -EINVAL;
1371
1372 /* invalidate the higher levels */
1373 g4x_invalidate_wms(crtc, wm_state, level);
1374
1375 /*
1376 * Determine if the FBC watermark(s) can be used. IF
1377 * this isn't the case we prefer to disable the FBC
1378 * watermark(s) rather than disable the SR/HPLL
1379 * level(s) entirely. 'level-1' is the highest valid
1380 * level here.
1381 */
1382 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1383
1384 return 0;
1385}
1386
1387static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1388 struct intel_crtc *crtc)
1389{
1390 struct intel_crtc_state *crtc_state =
1391 intel_atomic_get_new_crtc_state(state, crtc);
1392 const struct intel_plane_state *old_plane_state;
1393 const struct intel_plane_state *new_plane_state;
1394 struct intel_plane *plane;
1395 unsigned int dirty = 0;
1396 int i;
1397
1398 for_each_oldnew_intel_plane_in_state(state, plane,
1399 old_plane_state,
1400 new_plane_state, i) {
1401 if (new_plane_state->hw.crtc != &crtc->base &&
1402 old_plane_state->hw.crtc != &crtc->base)
1403 continue;
1404
1405 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1406 dirty |= BIT(plane->id);
1407 }
1408
1409 if (!dirty)
1410 return 0;
1411
1412 return _g4x_compute_pipe_wm(crtc_state);
1413}
1414
1415static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1416 struct intel_crtc *crtc)
1417{
1418 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1419 struct intel_crtc_state *new_crtc_state =
1420 intel_atomic_get_new_crtc_state(state, crtc);
1421 const struct intel_crtc_state *old_crtc_state =
1422 intel_atomic_get_old_crtc_state(state, crtc);
1423 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1424 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1425 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1426 enum plane_id plane_id;
1427
1428 if (!new_crtc_state->hw.active ||
1429 intel_crtc_needs_modeset(new_crtc_state)) {
1430 *intermediate = *optimal;
1431
1432 intermediate->cxsr = false;
1433 intermediate->hpll_en = false;
1434 goto out;
1435 }
1436
1437 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1438 !new_crtc_state->disable_cxsr;
1439 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1440 !new_crtc_state->disable_cxsr;
1441 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1442
1443 for_each_plane_id_on_crtc(crtc, plane_id) {
1444 intermediate->wm.plane[plane_id] =
1445 max(optimal->wm.plane[plane_id],
1446 active->wm.plane[plane_id]);
1447
1448 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1449 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1450 }
1451
1452 intermediate->sr.plane = max(optimal->sr.plane,
1453 active->sr.plane);
1454 intermediate->sr.cursor = max(optimal->sr.cursor,
1455 active->sr.cursor);
1456 intermediate->sr.fbc = max(optimal->sr.fbc,
1457 active->sr.fbc);
1458
1459 intermediate->hpll.plane = max(optimal->hpll.plane,
1460 active->hpll.plane);
1461 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1462 active->hpll.cursor);
1463 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1464 active->hpll.fbc);
1465
1466 drm_WARN_ON(&dev_priv->drm,
1467 (intermediate->sr.plane >
1468 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1469 intermediate->sr.cursor >
1470 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1471 intermediate->cxsr);
1472 drm_WARN_ON(&dev_priv->drm,
1473 (intermediate->sr.plane >
1474 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1475 intermediate->sr.cursor >
1476 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1477 intermediate->hpll_en);
1478
1479 drm_WARN_ON(&dev_priv->drm,
1480 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1481 intermediate->fbc_en && intermediate->cxsr);
1482 drm_WARN_ON(&dev_priv->drm,
1483 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1484 intermediate->fbc_en && intermediate->hpll_en);
1485
1486out:
1487 /*
1488 * If our intermediate WM are identical to the final WM, then we can
1489 * omit the post-vblank programming; only update if it's different.
1490 */
1491 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1492 new_crtc_state->wm.need_postvbl_update = true;
1493
1494 return 0;
1495}
1496
1497static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1498 struct g4x_wm_values *wm)
1499{
1500 struct intel_crtc *crtc;
1501 int num_active_pipes = 0;
1502
1503 wm->cxsr = true;
1504 wm->hpll_en = true;
1505 wm->fbc_en = true;
1506
1507 for_each_intel_crtc(&dev_priv->drm, crtc) {
1508 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1509
1510 if (!crtc->active)
1511 continue;
1512
1513 if (!wm_state->cxsr)
1514 wm->cxsr = false;
1515 if (!wm_state->hpll_en)
1516 wm->hpll_en = false;
1517 if (!wm_state->fbc_en)
1518 wm->fbc_en = false;
1519
1520 num_active_pipes++;
1521 }
1522
1523 if (num_active_pipes != 1) {
1524 wm->cxsr = false;
1525 wm->hpll_en = false;
1526 wm->fbc_en = false;
1527 }
1528
1529 for_each_intel_crtc(&dev_priv->drm, crtc) {
1530 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1531 enum pipe pipe = crtc->pipe;
1532
1533 wm->pipe[pipe] = wm_state->wm;
1534 if (crtc->active && wm->cxsr)
1535 wm->sr = wm_state->sr;
1536 if (crtc->active && wm->hpll_en)
1537 wm->hpll = wm_state->hpll;
1538 }
1539}
1540
1541static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1542{
1543 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1544 struct g4x_wm_values new_wm = {};
1545
1546 g4x_merge_wm(dev_priv, &new_wm);
1547
1548 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1549 return;
1550
1551 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1552 _intel_set_memory_cxsr(dev_priv, false);
1553
1554 g4x_write_wm_values(dev_priv, &new_wm);
1555
1556 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1557 _intel_set_memory_cxsr(dev_priv, true);
1558
1559 *old_wm = new_wm;
1560}
1561
1562static void g4x_initial_watermarks(struct intel_atomic_state *state,
1563 struct intel_crtc *crtc)
1564{
1565 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1566 const struct intel_crtc_state *crtc_state =
1567 intel_atomic_get_new_crtc_state(state, crtc);
1568
1569 mutex_lock(&dev_priv->display.wm.wm_mutex);
1570 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1571 g4x_program_watermarks(dev_priv);
1572 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1573}
1574
1575static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1576 struct intel_crtc *crtc)
1577{
1578 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1579 const struct intel_crtc_state *crtc_state =
1580 intel_atomic_get_new_crtc_state(state, crtc);
1581
1582 if (!crtc_state->wm.need_postvbl_update)
1583 return;
1584
1585 mutex_lock(&dev_priv->display.wm.wm_mutex);
1586 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1587 g4x_program_watermarks(dev_priv);
1588 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1589}
1590
1591/* latency must be in 0.1us units. */
1592static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1593 unsigned int htotal,
1594 unsigned int width,
1595 unsigned int cpp,
1596 unsigned int latency)
1597{
1598 unsigned int ret;
1599
1600 ret = intel_wm_method2(pixel_rate, htotal,
1601 width, cpp, latency);
1602 ret = DIV_ROUND_UP(ret, 64);
1603
1604 return ret;
1605}
1606
1607static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1608{
1609 /* all latencies in usec */
1610 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1611
1612 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1613
1614 if (IS_CHERRYVIEW(dev_priv)) {
1615 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1616 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1617
1618 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1619 }
1620}
1621
1622static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1623 const struct intel_plane_state *plane_state,
1624 int level)
1625{
1626 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1627 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1628 const struct drm_display_mode *pipe_mode =
1629 &crtc_state->hw.pipe_mode;
1630 unsigned int pixel_rate, htotal, cpp, width, wm;
1631
1632 if (dev_priv->display.wm.pri_latency[level] == 0)
1633 return USHRT_MAX;
1634
1635 if (!intel_wm_plane_visible(crtc_state, plane_state))
1636 return 0;
1637
1638 cpp = plane_state->hw.fb->format->cpp[0];
1639 pixel_rate = crtc_state->pixel_rate;
1640 htotal = pipe_mode->crtc_htotal;
1641 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1642
1643 if (plane->id == PLANE_CURSOR) {
1644 /*
1645 * FIXME the formula gives values that are
1646 * too big for the cursor FIFO, and hence we
1647 * would never be able to use cursors. For
1648 * now just hardcode the watermark.
1649 */
1650 wm = 63;
1651 } else {
1652 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1653 dev_priv->display.wm.pri_latency[level] * 10);
1654 }
1655
1656 return min_t(unsigned int, wm, USHRT_MAX);
1657}
1658
1659static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1660{
1661 return (active_planes & (BIT(PLANE_SPRITE0) |
1662 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1663}
1664
1665static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1666{
1667 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1668 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1669 const struct g4x_pipe_wm *raw =
1670 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1671 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1672 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1673 int num_active_planes = hweight8(active_planes);
1674 const int fifo_size = 511;
1675 int fifo_extra, fifo_left = fifo_size;
1676 int sprite0_fifo_extra = 0;
1677 unsigned int total_rate;
1678 enum plane_id plane_id;
1679
1680 /*
1681 * When enabling sprite0 after sprite1 has already been enabled
1682 * we tend to get an underrun unless sprite0 already has some
1683 * FIFO space allcoated. Hence we always allocate at least one
1684 * cacheline for sprite0 whenever sprite1 is enabled.
1685 *
1686 * All other plane enable sequences appear immune to this problem.
1687 */
1688 if (vlv_need_sprite0_fifo_workaround(active_planes))
1689 sprite0_fifo_extra = 1;
1690
1691 total_rate = raw->plane[PLANE_PRIMARY] +
1692 raw->plane[PLANE_SPRITE0] +
1693 raw->plane[PLANE_SPRITE1] +
1694 sprite0_fifo_extra;
1695
1696 if (total_rate > fifo_size)
1697 return -EINVAL;
1698
1699 if (total_rate == 0)
1700 total_rate = 1;
1701
1702 for_each_plane_id_on_crtc(crtc, plane_id) {
1703 unsigned int rate;
1704
1705 if ((active_planes & BIT(plane_id)) == 0) {
1706 fifo_state->plane[plane_id] = 0;
1707 continue;
1708 }
1709
1710 rate = raw->plane[plane_id];
1711 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1712 fifo_left -= fifo_state->plane[plane_id];
1713 }
1714
1715 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1716 fifo_left -= sprite0_fifo_extra;
1717
1718 fifo_state->plane[PLANE_CURSOR] = 63;
1719
1720 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1721
1722 /* spread the remainder evenly */
1723 for_each_plane_id_on_crtc(crtc, plane_id) {
1724 int plane_extra;
1725
1726 if (fifo_left == 0)
1727 break;
1728
1729 if ((active_planes & BIT(plane_id)) == 0)
1730 continue;
1731
1732 plane_extra = min(fifo_extra, fifo_left);
1733 fifo_state->plane[plane_id] += plane_extra;
1734 fifo_left -= plane_extra;
1735 }
1736
1737 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1738
1739 /* give it all to the first plane if none are active */
1740 if (active_planes == 0) {
1741 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1742 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1743 }
1744
1745 return 0;
1746}
1747
1748/* mark all levels starting from 'level' as invalid */
1749static void vlv_invalidate_wms(struct intel_crtc *crtc,
1750 struct vlv_wm_state *wm_state, int level)
1751{
1752 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1753
1754 for (; level < intel_wm_num_levels(dev_priv); level++) {
1755 enum plane_id plane_id;
1756
1757 for_each_plane_id_on_crtc(crtc, plane_id)
1758 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1759
1760 wm_state->sr[level].cursor = USHRT_MAX;
1761 wm_state->sr[level].plane = USHRT_MAX;
1762 }
1763}
1764
1765static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1766{
1767 if (wm > fifo_size)
1768 return USHRT_MAX;
1769 else
1770 return fifo_size - wm;
1771}
1772
1773/*
1774 * Starting from 'level' set all higher
1775 * levels to 'value' in the "raw" watermarks.
1776 */
1777static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1778 int level, enum plane_id plane_id, u16 value)
1779{
1780 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1781 int num_levels = intel_wm_num_levels(dev_priv);
1782 bool dirty = false;
1783
1784 for (; level < num_levels; level++) {
1785 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1786
1787 dirty |= raw->plane[plane_id] != value;
1788 raw->plane[plane_id] = value;
1789 }
1790
1791 return dirty;
1792}
1793
1794static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1795 const struct intel_plane_state *plane_state)
1796{
1797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1798 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799 enum plane_id plane_id = plane->id;
1800 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801 int level;
1802 bool dirty = false;
1803
1804 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1805 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806 goto out;
1807 }
1808
1809 for (level = 0; level < num_levels; level++) {
1810 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1811 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1812 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1813
1814 if (wm > max_wm)
1815 break;
1816
1817 dirty |= raw->plane[plane_id] != wm;
1818 raw->plane[plane_id] = wm;
1819 }
1820
1821 /* mark all higher levels as invalid */
1822 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823
1824out:
1825 if (dirty)
1826 drm_dbg_kms(&dev_priv->drm,
1827 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1828 plane->base.name,
1829 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1830 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1831 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1832
1833 return dirty;
1834}
1835
1836static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1837 enum plane_id plane_id, int level)
1838{
1839 const struct g4x_pipe_wm *raw =
1840 &crtc_state->wm.vlv.raw[level];
1841 const struct vlv_fifo_state *fifo_state =
1842 &crtc_state->wm.vlv.fifo_state;
1843
1844 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1845}
1846
1847static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1848{
1849 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1850 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1851 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1852 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1853}
1854
1855static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1856{
1857 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1858 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1860 const struct vlv_fifo_state *fifo_state =
1861 &crtc_state->wm.vlv.fifo_state;
1862 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1863 int num_active_planes = hweight8(active_planes);
1864 enum plane_id plane_id;
1865 int level;
1866
1867 /* initially allow all levels */
1868 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1869 /*
1870 * Note that enabling cxsr with no primary/sprite planes
1871 * enabled can wedge the pipe. Hence we only allow cxsr
1872 * with exactly one enabled primary/sprite plane.
1873 */
1874 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1875
1876 for (level = 0; level < wm_state->num_levels; level++) {
1877 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1878 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1879
1880 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1881 break;
1882
1883 for_each_plane_id_on_crtc(crtc, plane_id) {
1884 wm_state->wm[level].plane[plane_id] =
1885 vlv_invert_wm_value(raw->plane[plane_id],
1886 fifo_state->plane[plane_id]);
1887 }
1888
1889 wm_state->sr[level].plane =
1890 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1891 raw->plane[PLANE_SPRITE0],
1892 raw->plane[PLANE_SPRITE1]),
1893 sr_fifo_size);
1894
1895 wm_state->sr[level].cursor =
1896 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1897 63);
1898 }
1899
1900 if (level == 0)
1901 return -EINVAL;
1902
1903 /* limit to only levels we can actually handle */
1904 wm_state->num_levels = level;
1905
1906 /* invalidate the higher levels */
1907 vlv_invalidate_wms(crtc, wm_state, level);
1908
1909 return 0;
1910}
1911
1912static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1913 struct intel_crtc *crtc)
1914{
1915 struct intel_crtc_state *crtc_state =
1916 intel_atomic_get_new_crtc_state(state, crtc);
1917 const struct intel_plane_state *old_plane_state;
1918 const struct intel_plane_state *new_plane_state;
1919 struct intel_plane *plane;
1920 unsigned int dirty = 0;
1921 int i;
1922
1923 for_each_oldnew_intel_plane_in_state(state, plane,
1924 old_plane_state,
1925 new_plane_state, i) {
1926 if (new_plane_state->hw.crtc != &crtc->base &&
1927 old_plane_state->hw.crtc != &crtc->base)
1928 continue;
1929
1930 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1931 dirty |= BIT(plane->id);
1932 }
1933
1934 /*
1935 * DSPARB registers may have been reset due to the
1936 * power well being turned off. Make sure we restore
1937 * them to a consistent state even if no primary/sprite
1938 * planes are initially active. We also force a FIFO
1939 * recomputation so that we are sure to sanitize the
1940 * FIFO setting we took over from the BIOS even if there
1941 * are no active planes on the crtc.
1942 */
1943 if (intel_crtc_needs_modeset(crtc_state))
1944 dirty = ~0;
1945
1946 if (!dirty)
1947 return 0;
1948
1949 /* cursor changes don't warrant a FIFO recompute */
1950 if (dirty & ~BIT(PLANE_CURSOR)) {
1951 const struct intel_crtc_state *old_crtc_state =
1952 intel_atomic_get_old_crtc_state(state, crtc);
1953 const struct vlv_fifo_state *old_fifo_state =
1954 &old_crtc_state->wm.vlv.fifo_state;
1955 const struct vlv_fifo_state *new_fifo_state =
1956 &crtc_state->wm.vlv.fifo_state;
1957 int ret;
1958
1959 ret = vlv_compute_fifo(crtc_state);
1960 if (ret)
1961 return ret;
1962
1963 if (intel_crtc_needs_modeset(crtc_state) ||
1964 memcmp(old_fifo_state, new_fifo_state,
1965 sizeof(*new_fifo_state)) != 0)
1966 crtc_state->fifo_changed = true;
1967 }
1968
1969 return _vlv_compute_pipe_wm(crtc_state);
1970}
1971
1972#define VLV_FIFO(plane, value) \
1973 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1974
1975static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1976 struct intel_crtc *crtc)
1977{
1978 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1979 struct intel_uncore *uncore = &dev_priv->uncore;
1980 const struct intel_crtc_state *crtc_state =
1981 intel_atomic_get_new_crtc_state(state, crtc);
1982 const struct vlv_fifo_state *fifo_state =
1983 &crtc_state->wm.vlv.fifo_state;
1984 int sprite0_start, sprite1_start, fifo_size;
1985 u32 dsparb, dsparb2, dsparb3;
1986
1987 if (!crtc_state->fifo_changed)
1988 return;
1989
1990 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1991 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1992 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1993
1994 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1995 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1996
1997 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1998
1999 /*
2000 * uncore.lock serves a double purpose here. It allows us to
2001 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2002 * it protects the DSPARB registers from getting clobbered by
2003 * parallel updates from multiple pipes.
2004 *
2005 * intel_pipe_update_start() has already disabled interrupts
2006 * for us, so a plain spin_lock() is sufficient here.
2007 */
2008 spin_lock(&uncore->lock);
2009
2010 switch (crtc->pipe) {
2011 case PIPE_A:
2012 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2013 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2014
2015 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2016 VLV_FIFO(SPRITEB, 0xff));
2017 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2018 VLV_FIFO(SPRITEB, sprite1_start));
2019
2020 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2021 VLV_FIFO(SPRITEB_HI, 0x1));
2022 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2023 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2024
2025 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2026 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2027 break;
2028 case PIPE_B:
2029 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2030 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2031
2032 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2033 VLV_FIFO(SPRITED, 0xff));
2034 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2035 VLV_FIFO(SPRITED, sprite1_start));
2036
2037 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2038 VLV_FIFO(SPRITED_HI, 0xff));
2039 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2040 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2041
2042 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2043 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2044 break;
2045 case PIPE_C:
2046 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2047 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2048
2049 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2050 VLV_FIFO(SPRITEF, 0xff));
2051 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2052 VLV_FIFO(SPRITEF, sprite1_start));
2053
2054 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2055 VLV_FIFO(SPRITEF_HI, 0xff));
2056 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2057 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2058
2059 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2060 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2061 break;
2062 default:
2063 break;
2064 }
2065
2066 intel_uncore_posting_read_fw(uncore, DSPARB);
2067
2068 spin_unlock(&uncore->lock);
2069}
2070
2071#undef VLV_FIFO
2072
2073static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2074 struct intel_crtc *crtc)
2075{
2076 struct intel_crtc_state *new_crtc_state =
2077 intel_atomic_get_new_crtc_state(state, crtc);
2078 const struct intel_crtc_state *old_crtc_state =
2079 intel_atomic_get_old_crtc_state(state, crtc);
2080 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2081 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2082 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2083 int level;
2084
2085 if (!new_crtc_state->hw.active ||
2086 intel_crtc_needs_modeset(new_crtc_state)) {
2087 *intermediate = *optimal;
2088
2089 intermediate->cxsr = false;
2090 goto out;
2091 }
2092
2093 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2094 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2095 !new_crtc_state->disable_cxsr;
2096
2097 for (level = 0; level < intermediate->num_levels; level++) {
2098 enum plane_id plane_id;
2099
2100 for_each_plane_id_on_crtc(crtc, plane_id) {
2101 intermediate->wm[level].plane[plane_id] =
2102 min(optimal->wm[level].plane[plane_id],
2103 active->wm[level].plane[plane_id]);
2104 }
2105
2106 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2107 active->sr[level].plane);
2108 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2109 active->sr[level].cursor);
2110 }
2111
2112 vlv_invalidate_wms(crtc, intermediate, level);
2113
2114out:
2115 /*
2116 * If our intermediate WM are identical to the final WM, then we can
2117 * omit the post-vblank programming; only update if it's different.
2118 */
2119 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2120 new_crtc_state->wm.need_postvbl_update = true;
2121
2122 return 0;
2123}
2124
2125static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2126 struct vlv_wm_values *wm)
2127{
2128 struct intel_crtc *crtc;
2129 int num_active_pipes = 0;
2130
2131 wm->level = dev_priv->display.wm.max_level;
2132 wm->cxsr = true;
2133
2134 for_each_intel_crtc(&dev_priv->drm, crtc) {
2135 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2136
2137 if (!crtc->active)
2138 continue;
2139
2140 if (!wm_state->cxsr)
2141 wm->cxsr = false;
2142
2143 num_active_pipes++;
2144 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2145 }
2146
2147 if (num_active_pipes != 1)
2148 wm->cxsr = false;
2149
2150 if (num_active_pipes > 1)
2151 wm->level = VLV_WM_LEVEL_PM2;
2152
2153 for_each_intel_crtc(&dev_priv->drm, crtc) {
2154 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2155 enum pipe pipe = crtc->pipe;
2156
2157 wm->pipe[pipe] = wm_state->wm[wm->level];
2158 if (crtc->active && wm->cxsr)
2159 wm->sr = wm_state->sr[wm->level];
2160
2161 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2162 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2163 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2164 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2165 }
2166}
2167
2168static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2169{
2170 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2171 struct vlv_wm_values new_wm = {};
2172
2173 vlv_merge_wm(dev_priv, &new_wm);
2174
2175 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2176 return;
2177
2178 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2179 chv_set_memory_dvfs(dev_priv, false);
2180
2181 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2182 chv_set_memory_pm5(dev_priv, false);
2183
2184 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2185 _intel_set_memory_cxsr(dev_priv, false);
2186
2187 vlv_write_wm_values(dev_priv, &new_wm);
2188
2189 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2190 _intel_set_memory_cxsr(dev_priv, true);
2191
2192 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2193 chv_set_memory_pm5(dev_priv, true);
2194
2195 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2196 chv_set_memory_dvfs(dev_priv, true);
2197
2198 *old_wm = new_wm;
2199}
2200
2201static void vlv_initial_watermarks(struct intel_atomic_state *state,
2202 struct intel_crtc *crtc)
2203{
2204 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2205 const struct intel_crtc_state *crtc_state =
2206 intel_atomic_get_new_crtc_state(state, crtc);
2207
2208 mutex_lock(&dev_priv->display.wm.wm_mutex);
2209 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2210 vlv_program_watermarks(dev_priv);
2211 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2212}
2213
2214static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2215 struct intel_crtc *crtc)
2216{
2217 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2218 const struct intel_crtc_state *crtc_state =
2219 intel_atomic_get_new_crtc_state(state, crtc);
2220
2221 if (!crtc_state->wm.need_postvbl_update)
2222 return;
2223
2224 mutex_lock(&dev_priv->display.wm.wm_mutex);
2225 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2226 vlv_program_watermarks(dev_priv);
2227 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2228}
2229
2230static void i965_update_wm(struct drm_i915_private *dev_priv)
2231{
2232 struct intel_crtc *crtc;
2233 int srwm = 1;
2234 int cursor_sr = 16;
2235 bool cxsr_enabled;
2236
2237 /* Calc sr entries for one plane configs */
2238 crtc = single_enabled_crtc(dev_priv);
2239 if (crtc) {
2240 /* self-refresh has much higher latency */
2241 static const int sr_latency_ns = 12000;
2242 const struct drm_display_mode *pipe_mode =
2243 &crtc->config->hw.pipe_mode;
2244 const struct drm_framebuffer *fb =
2245 crtc->base.primary->state->fb;
2246 int pixel_rate = crtc->config->pixel_rate;
2247 int htotal = pipe_mode->crtc_htotal;
2248 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2249 int cpp = fb->format->cpp[0];
2250 int entries;
2251
2252 entries = intel_wm_method2(pixel_rate, htotal,
2253 width, cpp, sr_latency_ns / 100);
2254 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2255 srwm = I965_FIFO_SIZE - entries;
2256 if (srwm < 0)
2257 srwm = 1;
2258 srwm &= 0x1ff;
2259 drm_dbg_kms(&dev_priv->drm,
2260 "self-refresh entries: %d, wm: %d\n",
2261 entries, srwm);
2262
2263 entries = intel_wm_method2(pixel_rate, htotal,
2264 crtc->base.cursor->state->crtc_w, 4,
2265 sr_latency_ns / 100);
2266 entries = DIV_ROUND_UP(entries,
2267 i965_cursor_wm_info.cacheline_size) +
2268 i965_cursor_wm_info.guard_size;
2269
2270 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2271 if (cursor_sr > i965_cursor_wm_info.max_wm)
2272 cursor_sr = i965_cursor_wm_info.max_wm;
2273
2274 drm_dbg_kms(&dev_priv->drm,
2275 "self-refresh watermark: display plane %d "
2276 "cursor %d\n", srwm, cursor_sr);
2277
2278 cxsr_enabled = true;
2279 } else {
2280 cxsr_enabled = false;
2281 /* Turn off self refresh if both pipes are enabled */
2282 intel_set_memory_cxsr(dev_priv, false);
2283 }
2284
2285 drm_dbg_kms(&dev_priv->drm,
2286 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2287 srwm);
2288
2289 /* 965 has limitations... */
2290 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2291 FW_WM(8, CURSORB) |
2292 FW_WM(8, PLANEB) |
2293 FW_WM(8, PLANEA));
2294 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2295 FW_WM(8, PLANEC_OLD));
2296 /* update cursor SR watermark */
2297 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2298
2299 if (cxsr_enabled)
2300 intel_set_memory_cxsr(dev_priv, true);
2301}
2302
2303#undef FW_WM
2304
2305static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2306 enum i9xx_plane_id i9xx_plane)
2307{
2308 struct intel_plane *plane;
2309
2310 for_each_intel_plane(&i915->drm, plane) {
2311 if (plane->id == PLANE_PRIMARY &&
2312 plane->i9xx_plane == i9xx_plane)
2313 return intel_crtc_for_pipe(i915, plane->pipe);
2314 }
2315
2316 return NULL;
2317}
2318
2319static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2320{
2321 const struct intel_watermark_params *wm_info;
2322 u32 fwater_lo;
2323 u32 fwater_hi;
2324 int cwm, srwm = 1;
2325 int fifo_size;
2326 int planea_wm, planeb_wm;
2327 struct intel_crtc *crtc;
2328
2329 if (IS_I945GM(dev_priv))
2330 wm_info = &i945_wm_info;
2331 else if (DISPLAY_VER(dev_priv) != 2)
2332 wm_info = &i915_wm_info;
2333 else
2334 wm_info = &i830_a_wm_info;
2335
2336 if (DISPLAY_VER(dev_priv) == 2)
2337 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2338 else
2339 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2340 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2341 if (intel_crtc_active(crtc)) {
2342 const struct drm_framebuffer *fb =
2343 crtc->base.primary->state->fb;
2344 int cpp;
2345
2346 if (DISPLAY_VER(dev_priv) == 2)
2347 cpp = 4;
2348 else
2349 cpp = fb->format->cpp[0];
2350
2351 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2352 wm_info, fifo_size, cpp,
2353 pessimal_latency_ns);
2354 } else {
2355 planea_wm = fifo_size - wm_info->guard_size;
2356 if (planea_wm > (long)wm_info->max_wm)
2357 planea_wm = wm_info->max_wm;
2358 }
2359
2360 if (DISPLAY_VER(dev_priv) == 2)
2361 wm_info = &i830_bc_wm_info;
2362
2363 if (DISPLAY_VER(dev_priv) == 2)
2364 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2365 else
2366 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2367 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2368 if (intel_crtc_active(crtc)) {
2369 const struct drm_framebuffer *fb =
2370 crtc->base.primary->state->fb;
2371 int cpp;
2372
2373 if (DISPLAY_VER(dev_priv) == 2)
2374 cpp = 4;
2375 else
2376 cpp = fb->format->cpp[0];
2377
2378 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379 wm_info, fifo_size, cpp,
2380 pessimal_latency_ns);
2381 } else {
2382 planeb_wm = fifo_size - wm_info->guard_size;
2383 if (planeb_wm > (long)wm_info->max_wm)
2384 planeb_wm = wm_info->max_wm;
2385 }
2386
2387 drm_dbg_kms(&dev_priv->drm,
2388 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2389
2390 crtc = single_enabled_crtc(dev_priv);
2391 if (IS_I915GM(dev_priv) && crtc) {
2392 struct drm_i915_gem_object *obj;
2393
2394 obj = intel_fb_obj(crtc->base.primary->state->fb);
2395
2396 /* self-refresh seems busted with untiled */
2397 if (!i915_gem_object_is_tiled(obj))
2398 crtc = NULL;
2399 }
2400
2401 /*
2402 * Overlay gets an aggressive default since video jitter is bad.
2403 */
2404 cwm = 2;
2405
2406 /* Play safe and disable self-refresh before adjusting watermarks. */
2407 intel_set_memory_cxsr(dev_priv, false);
2408
2409 /* Calc sr entries for one plane configs */
2410 if (HAS_FW_BLC(dev_priv) && crtc) {
2411 /* self-refresh has much higher latency */
2412 static const int sr_latency_ns = 6000;
2413 const struct drm_display_mode *pipe_mode =
2414 &crtc->config->hw.pipe_mode;
2415 const struct drm_framebuffer *fb =
2416 crtc->base.primary->state->fb;
2417 int pixel_rate = crtc->config->pixel_rate;
2418 int htotal = pipe_mode->crtc_htotal;
2419 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2420 int cpp;
2421 int entries;
2422
2423 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2424 cpp = 4;
2425 else
2426 cpp = fb->format->cpp[0];
2427
2428 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2429 sr_latency_ns / 100);
2430 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2431 drm_dbg_kms(&dev_priv->drm,
2432 "self-refresh entries: %d\n", entries);
2433 srwm = wm_info->fifo_size - entries;
2434 if (srwm < 0)
2435 srwm = 1;
2436
2437 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2438 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2439 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2440 else
2441 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2442 }
2443
2444 drm_dbg_kms(&dev_priv->drm,
2445 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2446 planea_wm, planeb_wm, cwm, srwm);
2447
2448 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2449 fwater_hi = (cwm & 0x1f);
2450
2451 /* Set request length to 8 cachelines per fetch */
2452 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2453 fwater_hi = fwater_hi | (1 << 8);
2454
2455 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2456 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2457
2458 if (crtc)
2459 intel_set_memory_cxsr(dev_priv, true);
2460}
2461
2462static void i845_update_wm(struct drm_i915_private *dev_priv)
2463{
2464 struct intel_crtc *crtc;
2465 u32 fwater_lo;
2466 int planea_wm;
2467
2468 crtc = single_enabled_crtc(dev_priv);
2469 if (crtc == NULL)
2470 return;
2471
2472 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2473 &i845_wm_info,
2474 i845_get_fifo_size(dev_priv, PLANE_A),
2475 4, pessimal_latency_ns);
2476 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2477 fwater_lo |= (3<<8) | planea_wm;
2478
2479 drm_dbg_kms(&dev_priv->drm,
2480 "Setting FIFO watermarks - A: %d\n", planea_wm);
2481
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483}
2484
2485/* latency must be in 0.1us units. */
2486static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2487 unsigned int cpp,
2488 unsigned int latency)
2489{
2490 unsigned int ret;
2491
2492 ret = intel_wm_method1(pixel_rate, cpp, latency);
2493 ret = DIV_ROUND_UP(ret, 64) + 2;
2494
2495 return ret;
2496}
2497
2498/* latency must be in 0.1us units. */
2499static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2500 unsigned int htotal,
2501 unsigned int width,
2502 unsigned int cpp,
2503 unsigned int latency)
2504{
2505 unsigned int ret;
2506
2507 ret = intel_wm_method2(pixel_rate, htotal,
2508 width, cpp, latency);
2509 ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511 return ret;
2512}
2513
2514static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2515{
2516 /*
2517 * Neither of these should be possible since this function shouldn't be
2518 * called if the CRTC is off or the plane is invisible. But let's be
2519 * extra paranoid to avoid a potential divide-by-zero if we screw up
2520 * elsewhere in the driver.
2521 */
2522 if (WARN_ON(!cpp))
2523 return 0;
2524 if (WARN_ON(!horiz_pixels))
2525 return 0;
2526
2527 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2528}
2529
2530struct ilk_wm_maximums {
2531 u16 pri;
2532 u16 spr;
2533 u16 cur;
2534 u16 fbc;
2535};
2536
2537/*
2538 * For both WM_PIPE and WM_LP.
2539 * mem_value must be in 0.1us units.
2540 */
2541static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2542 const struct intel_plane_state *plane_state,
2543 u32 mem_value, bool is_lp)
2544{
2545 u32 method1, method2;
2546 int cpp;
2547
2548 if (mem_value == 0)
2549 return U32_MAX;
2550
2551 if (!intel_wm_plane_visible(crtc_state, plane_state))
2552 return 0;
2553
2554 cpp = plane_state->hw.fb->format->cpp[0];
2555
2556 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2557
2558 if (!is_lp)
2559 return method1;
2560
2561 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2562 crtc_state->hw.pipe_mode.crtc_htotal,
2563 drm_rect_width(&plane_state->uapi.src) >> 16,
2564 cpp, mem_value);
2565
2566 return min(method1, method2);
2567}
2568
2569/*
2570 * For both WM_PIPE and WM_LP.
2571 * mem_value must be in 0.1us units.
2572 */
2573static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2574 const struct intel_plane_state *plane_state,
2575 u32 mem_value)
2576{
2577 u32 method1, method2;
2578 int cpp;
2579
2580 if (mem_value == 0)
2581 return U32_MAX;
2582
2583 if (!intel_wm_plane_visible(crtc_state, plane_state))
2584 return 0;
2585
2586 cpp = plane_state->hw.fb->format->cpp[0];
2587
2588 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2589 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2590 crtc_state->hw.pipe_mode.crtc_htotal,
2591 drm_rect_width(&plane_state->uapi.src) >> 16,
2592 cpp, mem_value);
2593 return min(method1, method2);
2594}
2595
2596/*
2597 * For both WM_PIPE and WM_LP.
2598 * mem_value must be in 0.1us units.
2599 */
2600static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2601 const struct intel_plane_state *plane_state,
2602 u32 mem_value)
2603{
2604 int cpp;
2605
2606 if (mem_value == 0)
2607 return U32_MAX;
2608
2609 if (!intel_wm_plane_visible(crtc_state, plane_state))
2610 return 0;
2611
2612 cpp = plane_state->hw.fb->format->cpp[0];
2613
2614 return ilk_wm_method2(crtc_state->pixel_rate,
2615 crtc_state->hw.pipe_mode.crtc_htotal,
2616 drm_rect_width(&plane_state->uapi.src) >> 16,
2617 cpp, mem_value);
2618}
2619
2620/* Only for WM_LP. */
2621static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2622 const struct intel_plane_state *plane_state,
2623 u32 pri_val)
2624{
2625 int cpp;
2626
2627 if (!intel_wm_plane_visible(crtc_state, plane_state))
2628 return 0;
2629
2630 cpp = plane_state->hw.fb->format->cpp[0];
2631
2632 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2633 cpp);
2634}
2635
2636static unsigned int
2637ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2638{
2639 if (DISPLAY_VER(dev_priv) >= 8)
2640 return 3072;
2641 else if (DISPLAY_VER(dev_priv) >= 7)
2642 return 768;
2643 else
2644 return 512;
2645}
2646
2647static unsigned int
2648ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2649 int level, bool is_sprite)
2650{
2651 if (DISPLAY_VER(dev_priv) >= 8)
2652 /* BDW primary/sprite plane watermarks */
2653 return level == 0 ? 255 : 2047;
2654 else if (DISPLAY_VER(dev_priv) >= 7)
2655 /* IVB/HSW primary/sprite plane watermarks */
2656 return level == 0 ? 127 : 1023;
2657 else if (!is_sprite)
2658 /* ILK/SNB primary plane watermarks */
2659 return level == 0 ? 127 : 511;
2660 else
2661 /* ILK/SNB sprite plane watermarks */
2662 return level == 0 ? 63 : 255;
2663}
2664
2665static unsigned int
2666ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2667{
2668 if (DISPLAY_VER(dev_priv) >= 7)
2669 return level == 0 ? 63 : 255;
2670 else
2671 return level == 0 ? 31 : 63;
2672}
2673
2674static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2675{
2676 if (DISPLAY_VER(dev_priv) >= 8)
2677 return 31;
2678 else
2679 return 15;
2680}
2681
2682/* Calculate the maximum primary/sprite plane watermark */
2683static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2684 int level,
2685 const struct intel_wm_config *config,
2686 enum intel_ddb_partitioning ddb_partitioning,
2687 bool is_sprite)
2688{
2689 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2690
2691 /* if sprites aren't enabled, sprites get nothing */
2692 if (is_sprite && !config->sprites_enabled)
2693 return 0;
2694
2695 /* HSW allows LP1+ watermarks even with multiple pipes */
2696 if (level == 0 || config->num_pipes_active > 1) {
2697 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2698
2699 /*
2700 * For some reason the non self refresh
2701 * FIFO size is only half of the self
2702 * refresh FIFO size on ILK/SNB.
2703 */
2704 if (DISPLAY_VER(dev_priv) <= 6)
2705 fifo_size /= 2;
2706 }
2707
2708 if (config->sprites_enabled) {
2709 /* level 0 is always calculated with 1:1 split */
2710 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2711 if (is_sprite)
2712 fifo_size *= 5;
2713 fifo_size /= 6;
2714 } else {
2715 fifo_size /= 2;
2716 }
2717 }
2718
2719 /* clamp to max that the registers can hold */
2720 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2721}
2722
2723/* Calculate the maximum cursor plane watermark */
2724static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2725 int level,
2726 const struct intel_wm_config *config)
2727{
2728 /* HSW LP1+ watermarks w/ multiple pipes */
2729 if (level > 0 && config->num_pipes_active > 1)
2730 return 64;
2731
2732 /* otherwise just report max that registers can hold */
2733 return ilk_cursor_wm_reg_max(dev_priv, level);
2734}
2735
2736static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2737 int level,
2738 const struct intel_wm_config *config,
2739 enum intel_ddb_partitioning ddb_partitioning,
2740 struct ilk_wm_maximums *max)
2741{
2742 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2743 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2744 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2745 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2746}
2747
2748static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2749 int level,
2750 struct ilk_wm_maximums *max)
2751{
2752 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2753 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2754 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2755 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2756}
2757
2758static bool ilk_validate_wm_level(int level,
2759 const struct ilk_wm_maximums *max,
2760 struct intel_wm_level *result)
2761{
2762 bool ret;
2763
2764 /* already determined to be invalid? */
2765 if (!result->enable)
2766 return false;
2767
2768 result->enable = result->pri_val <= max->pri &&
2769 result->spr_val <= max->spr &&
2770 result->cur_val <= max->cur;
2771
2772 ret = result->enable;
2773
2774 /*
2775 * HACK until we can pre-compute everything,
2776 * and thus fail gracefully if LP0 watermarks
2777 * are exceeded...
2778 */
2779 if (level == 0 && !result->enable) {
2780 if (result->pri_val > max->pri)
2781 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2782 level, result->pri_val, max->pri);
2783 if (result->spr_val > max->spr)
2784 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2785 level, result->spr_val, max->spr);
2786 if (result->cur_val > max->cur)
2787 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2788 level, result->cur_val, max->cur);
2789
2790 result->pri_val = min_t(u32, result->pri_val, max->pri);
2791 result->spr_val = min_t(u32, result->spr_val, max->spr);
2792 result->cur_val = min_t(u32, result->cur_val, max->cur);
2793 result->enable = true;
2794 }
2795
2796 return ret;
2797}
2798
2799static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2800 const struct intel_crtc *crtc,
2801 int level,
2802 struct intel_crtc_state *crtc_state,
2803 const struct intel_plane_state *pristate,
2804 const struct intel_plane_state *sprstate,
2805 const struct intel_plane_state *curstate,
2806 struct intel_wm_level *result)
2807{
2808 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2809 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2810 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2811
2812 /* WM1+ latency values stored in 0.5us units */
2813 if (level > 0) {
2814 pri_latency *= 5;
2815 spr_latency *= 5;
2816 cur_latency *= 5;
2817 }
2818
2819 if (pristate) {
2820 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2821 pri_latency, level);
2822 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2823 }
2824
2825 if (sprstate)
2826 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2827
2828 if (curstate)
2829 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2830
2831 result->enable = true;
2832}
2833
2834static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2835{
2836 u64 sskpd;
2837
2838 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2839
2840 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2841 if (wm[0] == 0)
2842 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2843 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2844 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2845 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2846 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2847}
2848
2849static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2850{
2851 u32 sskpd;
2852
2853 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2854
2855 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2856 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2857 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2858 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2859}
2860
2861static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2862{
2863 u32 mltr;
2864
2865 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2866
2867 /* ILK primary LP0 latency is 700 ns */
2868 wm[0] = 7;
2869 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2870 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2871}
2872
2873static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2874 u16 wm[5])
2875{
2876 /* ILK sprite LP0 latency is 1300 ns */
2877 if (DISPLAY_VER(dev_priv) == 5)
2878 wm[0] = 13;
2879}
2880
2881static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2882 u16 wm[5])
2883{
2884 /* ILK cursor LP0 latency is 1300 ns */
2885 if (DISPLAY_VER(dev_priv) == 5)
2886 wm[0] = 13;
2887}
2888
2889int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2890{
2891 /* how many WM levels are we expecting */
2892 if (HAS_HW_SAGV_WM(dev_priv))
2893 return 5;
2894 else if (DISPLAY_VER(dev_priv) >= 9)
2895 return 7;
2896 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2897 return 4;
2898 else if (DISPLAY_VER(dev_priv) >= 6)
2899 return 3;
2900 else
2901 return 2;
2902}
2903
2904void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2905 const char *name, const u16 wm[])
2906{
2907 int level, max_level = ilk_wm_max_level(dev_priv);
2908
2909 for (level = 0; level <= max_level; level++) {
2910 unsigned int latency = wm[level];
2911
2912 if (latency == 0) {
2913 drm_dbg_kms(&dev_priv->drm,
2914 "%s WM%d latency not provided\n",
2915 name, level);
2916 continue;
2917 }
2918
2919 /*
2920 * - latencies are in us on gen9.
2921 * - before then, WM1+ latency values are in 0.5us units
2922 */
2923 if (DISPLAY_VER(dev_priv) >= 9)
2924 latency *= 10;
2925 else if (level > 0)
2926 latency *= 5;
2927
2928 drm_dbg_kms(&dev_priv->drm,
2929 "%s WM%d latency %u (%u.%u usec)\n", name, level,
2930 wm[level], latency / 10, latency % 10);
2931 }
2932}
2933
2934static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2935 u16 wm[5], u16 min)
2936{
2937 int level, max_level = ilk_wm_max_level(dev_priv);
2938
2939 if (wm[0] >= min)
2940 return false;
2941
2942 wm[0] = max(wm[0], min);
2943 for (level = 1; level <= max_level; level++)
2944 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2945
2946 return true;
2947}
2948
2949static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2950{
2951 bool changed;
2952
2953 /*
2954 * The BIOS provided WM memory latency values are often
2955 * inadequate for high resolution displays. Adjust them.
2956 */
2957 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2958 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2959 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2960
2961 if (!changed)
2962 return;
2963
2964 drm_dbg_kms(&dev_priv->drm,
2965 "WM latency values increased to avoid potential underruns\n");
2966 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2967 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2968 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2969}
2970
2971static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2972{
2973 /*
2974 * On some SNB machines (Thinkpad X220 Tablet at least)
2975 * LP3 usage can cause vblank interrupts to be lost.
2976 * The DEIIR bit will go high but it looks like the CPU
2977 * never gets interrupted.
2978 *
2979 * It's not clear whether other interrupt source could
2980 * be affected or if this is somehow limited to vblank
2981 * interrupts only. To play it safe we disable LP3
2982 * watermarks entirely.
2983 */
2984 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2985 dev_priv->display.wm.spr_latency[3] == 0 &&
2986 dev_priv->display.wm.cur_latency[3] == 0)
2987 return;
2988
2989 dev_priv->display.wm.pri_latency[3] = 0;
2990 dev_priv->display.wm.spr_latency[3] = 0;
2991 dev_priv->display.wm.cur_latency[3] = 0;
2992
2993 drm_dbg_kms(&dev_priv->drm,
2994 "LP3 watermarks disabled due to potential for lost interrupts\n");
2995 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2996 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2997 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2998}
2999
3000static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3001{
3002 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3003 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3005 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3006 else
3007 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3008
3009 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3010 sizeof(dev_priv->display.wm.pri_latency));
3011 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3012 sizeof(dev_priv->display.wm.pri_latency));
3013
3014 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3015 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3016
3017 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3018 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3019 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3020
3021 if (DISPLAY_VER(dev_priv) == 6) {
3022 snb_wm_latency_quirk(dev_priv);
3023 snb_wm_lp3_irq_quirk(dev_priv);
3024 }
3025}
3026
3027static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3028 struct intel_pipe_wm *pipe_wm)
3029{
3030 /* LP0 watermark maximums depend on this pipe alone */
3031 const struct intel_wm_config config = {
3032 .num_pipes_active = 1,
3033 .sprites_enabled = pipe_wm->sprites_enabled,
3034 .sprites_scaled = pipe_wm->sprites_scaled,
3035 };
3036 struct ilk_wm_maximums max;
3037
3038 /* LP0 watermarks always use 1/2 DDB partitioning */
3039 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3040
3041 /* At least LP0 must be valid */
3042 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3043 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3044 return false;
3045 }
3046
3047 return true;
3048}
3049
3050/* Compute new watermarks for the pipe */
3051static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3052 struct intel_crtc *crtc)
3053{
3054 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3055 struct intel_crtc_state *crtc_state =
3056 intel_atomic_get_new_crtc_state(state, crtc);
3057 struct intel_pipe_wm *pipe_wm;
3058 struct intel_plane *plane;
3059 const struct intel_plane_state *plane_state;
3060 const struct intel_plane_state *pristate = NULL;
3061 const struct intel_plane_state *sprstate = NULL;
3062 const struct intel_plane_state *curstate = NULL;
3063 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3064 struct ilk_wm_maximums max;
3065
3066 pipe_wm = &crtc_state->wm.ilk.optimal;
3067
3068 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3069 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3070 pristate = plane_state;
3071 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3072 sprstate = plane_state;
3073 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3074 curstate = plane_state;
3075 }
3076
3077 pipe_wm->pipe_enabled = crtc_state->hw.active;
3078 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3079 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3080
3081 usable_level = max_level;
3082
3083 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3084 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3085 usable_level = 1;
3086
3087 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3088 if (pipe_wm->sprites_scaled)
3089 usable_level = 0;
3090
3091 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3092 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3093 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3094
3095 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3096 return -EINVAL;
3097
3098 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3099
3100 for (level = 1; level <= usable_level; level++) {
3101 struct intel_wm_level *wm = &pipe_wm->wm[level];
3102
3103 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3104 pristate, sprstate, curstate, wm);
3105
3106 /*
3107 * Disable any watermark level that exceeds the
3108 * register maximums since such watermarks are
3109 * always invalid.
3110 */
3111 if (!ilk_validate_wm_level(level, &max, wm)) {
3112 memset(wm, 0, sizeof(*wm));
3113 break;
3114 }
3115 }
3116
3117 return 0;
3118}
3119
3120/*
3121 * Build a set of 'intermediate' watermark values that satisfy both the old
3122 * state and the new state. These can be programmed to the hardware
3123 * immediately.
3124 */
3125static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3126 struct intel_crtc *crtc)
3127{
3128 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3129 struct intel_crtc_state *new_crtc_state =
3130 intel_atomic_get_new_crtc_state(state, crtc);
3131 const struct intel_crtc_state *old_crtc_state =
3132 intel_atomic_get_old_crtc_state(state, crtc);
3133 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3134 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3135 int level, max_level = ilk_wm_max_level(dev_priv);
3136
3137 /*
3138 * Start with the final, target watermarks, then combine with the
3139 * currently active watermarks to get values that are safe both before
3140 * and after the vblank.
3141 */
3142 *a = new_crtc_state->wm.ilk.optimal;
3143 if (!new_crtc_state->hw.active ||
3144 intel_crtc_needs_modeset(new_crtc_state) ||
3145 state->skip_intermediate_wm)
3146 return 0;
3147
3148 a->pipe_enabled |= b->pipe_enabled;
3149 a->sprites_enabled |= b->sprites_enabled;
3150 a->sprites_scaled |= b->sprites_scaled;
3151
3152 for (level = 0; level <= max_level; level++) {
3153 struct intel_wm_level *a_wm = &a->wm[level];
3154 const struct intel_wm_level *b_wm = &b->wm[level];
3155
3156 a_wm->enable &= b_wm->enable;
3157 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3158 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3159 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3160 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3161 }
3162
3163 /*
3164 * We need to make sure that these merged watermark values are
3165 * actually a valid configuration themselves. If they're not,
3166 * there's no safe way to transition from the old state to
3167 * the new state, so we need to fail the atomic transaction.
3168 */
3169 if (!ilk_validate_pipe_wm(dev_priv, a))
3170 return -EINVAL;
3171
3172 /*
3173 * If our intermediate WM are identical to the final WM, then we can
3174 * omit the post-vblank programming; only update if it's different.
3175 */
3176 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3177 new_crtc_state->wm.need_postvbl_update = true;
3178
3179 return 0;
3180}
3181
3182/*
3183 * Merge the watermarks from all active pipes for a specific level.
3184 */
3185static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3186 int level,
3187 struct intel_wm_level *ret_wm)
3188{
3189 const struct intel_crtc *crtc;
3190
3191 ret_wm->enable = true;
3192
3193 for_each_intel_crtc(&dev_priv->drm, crtc) {
3194 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3195 const struct intel_wm_level *wm = &active->wm[level];
3196
3197 if (!active->pipe_enabled)
3198 continue;
3199
3200 /*
3201 * The watermark values may have been used in the past,
3202 * so we must maintain them in the registers for some
3203 * time even if the level is now disabled.
3204 */
3205 if (!wm->enable)
3206 ret_wm->enable = false;
3207
3208 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3209 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3210 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3211 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3212 }
3213}
3214
3215/*
3216 * Merge all low power watermarks for all active pipes.
3217 */
3218static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3219 const struct intel_wm_config *config,
3220 const struct ilk_wm_maximums *max,
3221 struct intel_pipe_wm *merged)
3222{
3223 int level, max_level = ilk_wm_max_level(dev_priv);
3224 int last_enabled_level = max_level;
3225
3226 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3227 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3228 config->num_pipes_active > 1)
3229 last_enabled_level = 0;
3230
3231 /* ILK: FBC WM must be disabled always */
3232 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3233
3234 /* merge each WM1+ level */
3235 for (level = 1; level <= max_level; level++) {
3236 struct intel_wm_level *wm = &merged->wm[level];
3237
3238 ilk_merge_wm_level(dev_priv, level, wm);
3239
3240 if (level > last_enabled_level)
3241 wm->enable = false;
3242 else if (!ilk_validate_wm_level(level, max, wm))
3243 /* make sure all following levels get disabled */
3244 last_enabled_level = level - 1;
3245
3246 /*
3247 * The spec says it is preferred to disable
3248 * FBC WMs instead of disabling a WM level.
3249 */
3250 if (wm->fbc_val > max->fbc) {
3251 if (wm->enable)
3252 merged->fbc_wm_enabled = false;
3253 wm->fbc_val = 0;
3254 }
3255 }
3256
3257 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3258 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3259 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3260 for (level = 2; level <= max_level; level++) {
3261 struct intel_wm_level *wm = &merged->wm[level];
3262
3263 wm->enable = false;
3264 }
3265 }
3266}
3267
3268static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3269{
3270 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3271 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3272}
3273
3274/* The value we need to program into the WM_LPx latency field */
3275static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3276 int level)
3277{
3278 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3279 return 2 * level;
3280 else
3281 return dev_priv->display.wm.pri_latency[level];
3282}
3283
3284static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3285 const struct intel_pipe_wm *merged,
3286 enum intel_ddb_partitioning partitioning,
3287 struct ilk_wm_values *results)
3288{
3289 struct intel_crtc *crtc;
3290 int level, wm_lp;
3291
3292 results->enable_fbc_wm = merged->fbc_wm_enabled;
3293 results->partitioning = partitioning;
3294
3295 /* LP1+ register values */
3296 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3297 const struct intel_wm_level *r;
3298
3299 level = ilk_wm_lp_to_level(wm_lp, merged);
3300
3301 r = &merged->wm[level];
3302
3303 /*
3304 * Maintain the watermark values even if the level is
3305 * disabled. Doing otherwise could cause underruns.
3306 */
3307 results->wm_lp[wm_lp - 1] =
3308 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3309 WM_LP_PRIMARY(r->pri_val) |
3310 WM_LP_CURSOR(r->cur_val);
3311
3312 if (r->enable)
3313 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3314
3315 if (DISPLAY_VER(dev_priv) >= 8)
3316 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3317 else
3318 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3319
3320 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3321
3322 /*
3323 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3324 * level is disabled. Doing otherwise could cause underruns.
3325 */
3326 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3327 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3328 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3329 }
3330 }
3331
3332 /* LP0 register values */
3333 for_each_intel_crtc(&dev_priv->drm, crtc) {
3334 enum pipe pipe = crtc->pipe;
3335 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3336 const struct intel_wm_level *r = &pipe_wm->wm[0];
3337
3338 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3339 continue;
3340
3341 results->wm_pipe[pipe] =
3342 WM0_PIPE_PRIMARY(r->pri_val) |
3343 WM0_PIPE_SPRITE(r->spr_val) |
3344 WM0_PIPE_CURSOR(r->cur_val);
3345 }
3346}
3347
3348/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3349 * case both are at the same level. Prefer r1 in case they're the same. */
3350static struct intel_pipe_wm *
3351ilk_find_best_result(struct drm_i915_private *dev_priv,
3352 struct intel_pipe_wm *r1,
3353 struct intel_pipe_wm *r2)
3354{
3355 int level, max_level = ilk_wm_max_level(dev_priv);
3356 int level1 = 0, level2 = 0;
3357
3358 for (level = 1; level <= max_level; level++) {
3359 if (r1->wm[level].enable)
3360 level1 = level;
3361 if (r2->wm[level].enable)
3362 level2 = level;
3363 }
3364
3365 if (level1 == level2) {
3366 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3367 return r2;
3368 else
3369 return r1;
3370 } else if (level1 > level2) {
3371 return r1;
3372 } else {
3373 return r2;
3374 }
3375}
3376
3377/* dirty bits used to track which watermarks need changes */
3378#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3379#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3380#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3381#define WM_DIRTY_FBC (1 << 24)
3382#define WM_DIRTY_DDB (1 << 25)
3383
3384static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3385 const struct ilk_wm_values *old,
3386 const struct ilk_wm_values *new)
3387{
3388 unsigned int dirty = 0;
3389 enum pipe pipe;
3390 int wm_lp;
3391
3392 for_each_pipe(dev_priv, pipe) {
3393 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3394 dirty |= WM_DIRTY_PIPE(pipe);
3395 /* Must disable LP1+ watermarks too */
3396 dirty |= WM_DIRTY_LP_ALL;
3397 }
3398 }
3399
3400 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3401 dirty |= WM_DIRTY_FBC;
3402 /* Must disable LP1+ watermarks too */
3403 dirty |= WM_DIRTY_LP_ALL;
3404 }
3405
3406 if (old->partitioning != new->partitioning) {
3407 dirty |= WM_DIRTY_DDB;
3408 /* Must disable LP1+ watermarks too */
3409 dirty |= WM_DIRTY_LP_ALL;
3410 }
3411
3412 /* LP1+ watermarks already deemed dirty, no need to continue */
3413 if (dirty & WM_DIRTY_LP_ALL)
3414 return dirty;
3415
3416 /* Find the lowest numbered LP1+ watermark in need of an update... */
3417 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3418 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3419 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3420 break;
3421 }
3422
3423 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3424 for (; wm_lp <= 3; wm_lp++)
3425 dirty |= WM_DIRTY_LP(wm_lp);
3426
3427 return dirty;
3428}
3429
3430static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3431 unsigned int dirty)
3432{
3433 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3434 bool changed = false;
3435
3436 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3437 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3438 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3439 changed = true;
3440 }
3441 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3442 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3443 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3444 changed = true;
3445 }
3446 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3447 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3448 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3449 changed = true;
3450 }
3451
3452 /*
3453 * Don't touch WM_LP_SPRITE_ENABLE here.
3454 * Doing so could cause underruns.
3455 */
3456
3457 return changed;
3458}
3459
3460/*
3461 * The spec says we shouldn't write when we don't need, because every write
3462 * causes WMs to be re-evaluated, expending some power.
3463 */
3464static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3465 struct ilk_wm_values *results)
3466{
3467 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3468 unsigned int dirty;
3469
3470 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3471 if (!dirty)
3472 return;
3473
3474 _ilk_disable_lp_wm(dev_priv, dirty);
3475
3476 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3477 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3478 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3479 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3480 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3481 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3482
3483 if (dirty & WM_DIRTY_DDB) {
3484 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3485 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3486 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3487 WM_MISC_DATA_PARTITION_5_6);
3488 else
3489 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3490 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3491 DISP_DATA_PARTITION_5_6);
3492 }
3493
3494 if (dirty & WM_DIRTY_FBC)
3495 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3496 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3497
3498 if (dirty & WM_DIRTY_LP(1) &&
3499 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3500 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3501
3502 if (DISPLAY_VER(dev_priv) >= 7) {
3503 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3504 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3505 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3506 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3507 }
3508
3509 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3510 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3511 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3512 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3513 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3514 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3515
3516 dev_priv->display.wm.hw = *results;
3517}
3518
3519bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3520{
3521 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3522}
3523
3524static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3525 struct intel_wm_config *config)
3526{
3527 struct intel_crtc *crtc;
3528
3529 /* Compute the currently _active_ config */
3530 for_each_intel_crtc(&dev_priv->drm, crtc) {
3531 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3532
3533 if (!wm->pipe_enabled)
3534 continue;
3535
3536 config->sprites_enabled |= wm->sprites_enabled;
3537 config->sprites_scaled |= wm->sprites_scaled;
3538 config->num_pipes_active++;
3539 }
3540}
3541
3542static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3543{
3544 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3545 struct ilk_wm_maximums max;
3546 struct intel_wm_config config = {};
3547 struct ilk_wm_values results = {};
3548 enum intel_ddb_partitioning partitioning;
3549
3550 ilk_compute_wm_config(dev_priv, &config);
3551
3552 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3553 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3554
3555 /* 5/6 split only in single pipe config on IVB+ */
3556 if (DISPLAY_VER(dev_priv) >= 7 &&
3557 config.num_pipes_active == 1 && config.sprites_enabled) {
3558 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3559 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3560
3561 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3562 } else {
3563 best_lp_wm = &lp_wm_1_2;
3564 }
3565
3566 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3567 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3568
3569 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3570
3571 ilk_write_wm_values(dev_priv, &results);
3572}
3573
3574static void ilk_initial_watermarks(struct intel_atomic_state *state,
3575 struct intel_crtc *crtc)
3576{
3577 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3578 const struct intel_crtc_state *crtc_state =
3579 intel_atomic_get_new_crtc_state(state, crtc);
3580
3581 mutex_lock(&dev_priv->display.wm.wm_mutex);
3582 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3583 ilk_program_watermarks(dev_priv);
3584 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3585}
3586
3587static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3588 struct intel_crtc *crtc)
3589{
3590 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3591 const struct intel_crtc_state *crtc_state =
3592 intel_atomic_get_new_crtc_state(state, crtc);
3593
3594 if (!crtc_state->wm.need_postvbl_update)
3595 return;
3596
3597 mutex_lock(&dev_priv->display.wm.wm_mutex);
3598 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3599 ilk_program_watermarks(dev_priv);
3600 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3601}
3602
3603static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3604{
3605 struct drm_device *dev = crtc->base.dev;
3606 struct drm_i915_private *dev_priv = to_i915(dev);
3607 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3608 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3609 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3610 enum pipe pipe = crtc->pipe;
3611
3612 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3613
3614 memset(active, 0, sizeof(*active));
3615
3616 active->pipe_enabled = crtc->active;
3617
3618 if (active->pipe_enabled) {
3619 u32 tmp = hw->wm_pipe[pipe];
3620
3621 /*
3622 * For active pipes LP0 watermark is marked as
3623 * enabled, and LP1+ watermaks as disabled since
3624 * we can't really reverse compute them in case
3625 * multiple pipes are active.
3626 */
3627 active->wm[0].enable = true;
3628 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3629 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3630 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3631 } else {
3632 int level, max_level = ilk_wm_max_level(dev_priv);
3633
3634 /*
3635 * For inactive pipes, all watermark levels
3636 * should be marked as enabled but zeroed,
3637 * which is what we'd compute them to.
3638 */
3639 for (level = 0; level <= max_level; level++)
3640 active->wm[level].enable = true;
3641 }
3642
3643 crtc->wm.active.ilk = *active;
3644}
3645
3646#define _FW_WM(value, plane) \
3647 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3648#define _FW_WM_VLV(value, plane) \
3649 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3650
3651static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3652 struct g4x_wm_values *wm)
3653{
3654 u32 tmp;
3655
3656 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3657 wm->sr.plane = _FW_WM(tmp, SR);
3658 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3659 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3660 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3661
3662 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3663 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3664 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3665 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3666 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3667 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3668 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3669
3670 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3671 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3672 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3673 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3674 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3675}
3676
3677static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3678 struct vlv_wm_values *wm)
3679{
3680 enum pipe pipe;
3681 u32 tmp;
3682
3683 for_each_pipe(dev_priv, pipe) {
3684 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3685
3686 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3687 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3688 wm->ddl[pipe].plane[PLANE_CURSOR] =
3689 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3690 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3691 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3692 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3693 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3694 }
3695
3696 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3697 wm->sr.plane = _FW_WM(tmp, SR);
3698 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3699 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3700 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3701
3702 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3703 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3704 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3705 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3706
3707 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3708 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3709
3710 if (IS_CHERRYVIEW(dev_priv)) {
3711 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3712 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3713 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3714
3715 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3716 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3717 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3718
3719 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3720 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3721 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3722
3723 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3724 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3725 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3726 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3727 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3728 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3729 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3730 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3731 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3732 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3733 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3734 } else {
3735 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3736 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3737 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3738
3739 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3740 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3741 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3742 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3743 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3744 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3745 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3746 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3747 }
3748}
3749
3750#undef _FW_WM
3751#undef _FW_WM_VLV
3752
3753void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3754{
3755 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3756 struct intel_crtc *crtc;
3757
3758 g4x_read_wm_values(dev_priv, wm);
3759
3760 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3761
3762 for_each_intel_crtc(&dev_priv->drm, crtc) {
3763 struct intel_crtc_state *crtc_state =
3764 to_intel_crtc_state(crtc->base.state);
3765 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3766 struct g4x_pipe_wm *raw;
3767 enum pipe pipe = crtc->pipe;
3768 enum plane_id plane_id;
3769 int level, max_level;
3770
3771 active->cxsr = wm->cxsr;
3772 active->hpll_en = wm->hpll_en;
3773 active->fbc_en = wm->fbc_en;
3774
3775 active->sr = wm->sr;
3776 active->hpll = wm->hpll;
3777
3778 for_each_plane_id_on_crtc(crtc, plane_id) {
3779 active->wm.plane[plane_id] =
3780 wm->pipe[pipe].plane[plane_id];
3781 }
3782
3783 if (wm->cxsr && wm->hpll_en)
3784 max_level = G4X_WM_LEVEL_HPLL;
3785 else if (wm->cxsr)
3786 max_level = G4X_WM_LEVEL_SR;
3787 else
3788 max_level = G4X_WM_LEVEL_NORMAL;
3789
3790 level = G4X_WM_LEVEL_NORMAL;
3791 raw = &crtc_state->wm.g4x.raw[level];
3792 for_each_plane_id_on_crtc(crtc, plane_id)
3793 raw->plane[plane_id] = active->wm.plane[plane_id];
3794
3795 level = G4X_WM_LEVEL_SR;
3796 if (level > max_level)
3797 goto out;
3798
3799 raw = &crtc_state->wm.g4x.raw[level];
3800 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3801 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3802 raw->plane[PLANE_SPRITE0] = 0;
3803 raw->fbc = active->sr.fbc;
3804
3805 level = G4X_WM_LEVEL_HPLL;
3806 if (level > max_level)
3807 goto out;
3808
3809 raw = &crtc_state->wm.g4x.raw[level];
3810 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3811 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3812 raw->plane[PLANE_SPRITE0] = 0;
3813 raw->fbc = active->hpll.fbc;
3814
3815 level++;
3816 out:
3817 for_each_plane_id_on_crtc(crtc, plane_id)
3818 g4x_raw_plane_wm_set(crtc_state, level,
3819 plane_id, USHRT_MAX);
3820 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3821
3822 g4x_invalidate_wms(crtc, active, level);
3823
3824 crtc_state->wm.g4x.optimal = *active;
3825 crtc_state->wm.g4x.intermediate = *active;
3826
3827 drm_dbg_kms(&dev_priv->drm,
3828 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3829 pipe_name(pipe),
3830 wm->pipe[pipe].plane[PLANE_PRIMARY],
3831 wm->pipe[pipe].plane[PLANE_CURSOR],
3832 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3833 }
3834
3835 drm_dbg_kms(&dev_priv->drm,
3836 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3837 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3838 drm_dbg_kms(&dev_priv->drm,
3839 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3840 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3841 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3842 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3843 str_yes_no(wm->fbc_en));
3844}
3845
3846void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3847{
3848 struct intel_plane *plane;
3849 struct intel_crtc *crtc;
3850
3851 mutex_lock(&dev_priv->display.wm.wm_mutex);
3852
3853 for_each_intel_plane(&dev_priv->drm, plane) {
3854 struct intel_crtc *crtc =
3855 intel_crtc_for_pipe(dev_priv, plane->pipe);
3856 struct intel_crtc_state *crtc_state =
3857 to_intel_crtc_state(crtc->base.state);
3858 struct intel_plane_state *plane_state =
3859 to_intel_plane_state(plane->base.state);
3860 enum plane_id plane_id = plane->id;
3861 int level, num_levels = intel_wm_num_levels(dev_priv);
3862
3863 if (plane_state->uapi.visible)
3864 continue;
3865
3866 for (level = 0; level < num_levels; level++) {
3867 struct g4x_pipe_wm *raw =
3868 &crtc_state->wm.g4x.raw[level];
3869
3870 raw->plane[plane_id] = 0;
3871
3872 if (plane_id == PLANE_PRIMARY)
3873 raw->fbc = 0;
3874 }
3875 }
3876
3877 for_each_intel_crtc(&dev_priv->drm, crtc) {
3878 struct intel_crtc_state *crtc_state =
3879 to_intel_crtc_state(crtc->base.state);
3880 int ret;
3881
3882 ret = _g4x_compute_pipe_wm(crtc_state);
3883 drm_WARN_ON(&dev_priv->drm, ret);
3884
3885 crtc_state->wm.g4x.intermediate =
3886 crtc_state->wm.g4x.optimal;
3887 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3888 }
3889
3890 g4x_program_watermarks(dev_priv);
3891
3892 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3893}
3894
3895void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3896{
3897 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3898 struct intel_crtc *crtc;
3899 u32 val;
3900
3901 vlv_read_wm_values(dev_priv, wm);
3902
3903 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3904 wm->level = VLV_WM_LEVEL_PM2;
3905
3906 if (IS_CHERRYVIEW(dev_priv)) {
3907 vlv_punit_get(dev_priv);
3908
3909 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3910 if (val & DSP_MAXFIFO_PM5_ENABLE)
3911 wm->level = VLV_WM_LEVEL_PM5;
3912
3913 /*
3914 * If DDR DVFS is disabled in the BIOS, Punit
3915 * will never ack the request. So if that happens
3916 * assume we don't have to enable/disable DDR DVFS
3917 * dynamically. To test that just set the REQ_ACK
3918 * bit to poke the Punit, but don't change the
3919 * HIGH/LOW bits so that we don't actually change
3920 * the current state.
3921 */
3922 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3923 val |= FORCE_DDR_FREQ_REQ_ACK;
3924 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3925
3926 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3927 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3928 drm_dbg_kms(&dev_priv->drm,
3929 "Punit not acking DDR DVFS request, "
3930 "assuming DDR DVFS is disabled\n");
3931 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3932 } else {
3933 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3934 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3935 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3936 }
3937
3938 vlv_punit_put(dev_priv);
3939 }
3940
3941 for_each_intel_crtc(&dev_priv->drm, crtc) {
3942 struct intel_crtc_state *crtc_state =
3943 to_intel_crtc_state(crtc->base.state);
3944 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3945 const struct vlv_fifo_state *fifo_state =
3946 &crtc_state->wm.vlv.fifo_state;
3947 enum pipe pipe = crtc->pipe;
3948 enum plane_id plane_id;
3949 int level;
3950
3951 vlv_get_fifo_size(crtc_state);
3952
3953 active->num_levels = wm->level + 1;
3954 active->cxsr = wm->cxsr;
3955
3956 for (level = 0; level < active->num_levels; level++) {
3957 struct g4x_pipe_wm *raw =
3958 &crtc_state->wm.vlv.raw[level];
3959
3960 active->sr[level].plane = wm->sr.plane;
3961 active->sr[level].cursor = wm->sr.cursor;
3962
3963 for_each_plane_id_on_crtc(crtc, plane_id) {
3964 active->wm[level].plane[plane_id] =
3965 wm->pipe[pipe].plane[plane_id];
3966
3967 raw->plane[plane_id] =
3968 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3969 fifo_state->plane[plane_id]);
3970 }
3971 }
3972
3973 for_each_plane_id_on_crtc(crtc, plane_id)
3974 vlv_raw_plane_wm_set(crtc_state, level,
3975 plane_id, USHRT_MAX);
3976 vlv_invalidate_wms(crtc, active, level);
3977
3978 crtc_state->wm.vlv.optimal = *active;
3979 crtc_state->wm.vlv.intermediate = *active;
3980
3981 drm_dbg_kms(&dev_priv->drm,
3982 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3983 pipe_name(pipe),
3984 wm->pipe[pipe].plane[PLANE_PRIMARY],
3985 wm->pipe[pipe].plane[PLANE_CURSOR],
3986 wm->pipe[pipe].plane[PLANE_SPRITE0],
3987 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3988 }
3989
3990 drm_dbg_kms(&dev_priv->drm,
3991 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3992 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3993}
3994
3995void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3996{
3997 struct intel_plane *plane;
3998 struct intel_crtc *crtc;
3999
4000 mutex_lock(&dev_priv->display.wm.wm_mutex);
4001
4002 for_each_intel_plane(&dev_priv->drm, plane) {
4003 struct intel_crtc *crtc =
4004 intel_crtc_for_pipe(dev_priv, plane->pipe);
4005 struct intel_crtc_state *crtc_state =
4006 to_intel_crtc_state(crtc->base.state);
4007 struct intel_plane_state *plane_state =
4008 to_intel_plane_state(plane->base.state);
4009 enum plane_id plane_id = plane->id;
4010 int level, num_levels = intel_wm_num_levels(dev_priv);
4011
4012 if (plane_state->uapi.visible)
4013 continue;
4014
4015 for (level = 0; level < num_levels; level++) {
4016 struct g4x_pipe_wm *raw =
4017 &crtc_state->wm.vlv.raw[level];
4018
4019 raw->plane[plane_id] = 0;
4020 }
4021 }
4022
4023 for_each_intel_crtc(&dev_priv->drm, crtc) {
4024 struct intel_crtc_state *crtc_state =
4025 to_intel_crtc_state(crtc->base.state);
4026 int ret;
4027
4028 ret = _vlv_compute_pipe_wm(crtc_state);
4029 drm_WARN_ON(&dev_priv->drm, ret);
4030
4031 crtc_state->wm.vlv.intermediate =
4032 crtc_state->wm.vlv.optimal;
4033 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4034 }
4035
4036 vlv_program_watermarks(dev_priv);
4037
4038 mutex_unlock(&dev_priv->display.wm.wm_mutex);
4039}
4040
4041/*
4042 * FIXME should probably kill this and improve
4043 * the real watermark readout/sanitation instead
4044 */
4045static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4046{
4047 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4048 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4049 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4050
4051 /*
4052 * Don't touch WM_LP_SPRITE_ENABLE here.
4053 * Doing so could cause underruns.
4054 */
4055}
4056
4057void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4058{
4059 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4060 struct intel_crtc *crtc;
4061
4062 ilk_init_lp_watermarks(dev_priv);
4063
4064 for_each_intel_crtc(&dev_priv->drm, crtc)
4065 ilk_pipe_wm_get_hw_state(crtc);
4066
4067 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4068 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4069 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4070
4071 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4072 if (DISPLAY_VER(dev_priv) >= 7) {
4073 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4074 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4075 }
4076
4077 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4078 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4079 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4080 else if (IS_IVYBRIDGE(dev_priv))
4081 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4082 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4083
4084 hw->enable_fbc_wm =
4085 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4086}
4087
4088static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4089{
4090 /*
4091 * On Ibex Peak and Cougar Point, we need to disable clock
4092 * gating for the panel power sequencer or it will fail to
4093 * start up when no ports are active.
4094 */
4095 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4096}
4097
4098static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4099{
4100 enum pipe pipe;
4101
4102 for_each_pipe(dev_priv, pipe) {
4103 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4104
4105 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4106 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4107 }
4108}
4109
4110static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4111{
4112 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4113
4114 /*
4115 * Required for FBC
4116 * WaFbcDisableDpfcClockGating:ilk
4117 */
4118 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4119 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4120 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4121
4122 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4123 MARIUNIT_CLOCK_GATE_DISABLE |
4124 SVSMUNIT_CLOCK_GATE_DISABLE);
4125 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4126 VFMUNIT_CLOCK_GATE_DISABLE);
4127
4128 /*
4129 * According to the spec the following bits should be set in
4130 * order to enable memory self-refresh
4131 * The bit 22/21 of 0x42004
4132 * The bit 5 of 0x42020
4133 * The bit 15 of 0x45000
4134 */
4135 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4136 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4137 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4138 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4139 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4140 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4141 DISP_FBC_WM_DIS));
4142
4143 /*
4144 * Based on the document from hardware guys the following bits
4145 * should be set unconditionally in order to enable FBC.
4146 * The bit 22 of 0x42000
4147 * The bit 22 of 0x42004
4148 * The bit 7,8,9 of 0x42020.
4149 */
4150 if (IS_IRONLAKE_M(dev_priv)) {
4151 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4152 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4153 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4154 }
4155
4156 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4157
4158 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4159
4160 g4x_disable_trickle_feed(dev_priv);
4161
4162 ibx_init_clock_gating(dev_priv);
4163}
4164
4165static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4166{
4167 enum pipe pipe;
4168 u32 val;
4169
4170 /*
4171 * On Ibex Peak and Cougar Point, we need to disable clock
4172 * gating for the panel power sequencer or it will fail to
4173 * start up when no ports are active.
4174 */
4175 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4176 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4177 PCH_CPUNIT_CLOCK_GATE_DISABLE);
4178 intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4179 /* The below fixes the weird display corruption, a few pixels shifted
4180 * downward, on (only) LVDS of some HP laptops with IVY.
4181 */
4182 for_each_pipe(dev_priv, pipe) {
4183 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4184 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4185 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4186 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4187 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4188 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4189 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4190 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4191 }
4192 /* WADP0ClockGatingDisable */
4193 for_each_pipe(dev_priv, pipe) {
4194 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4195 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4196 }
4197}
4198
4199static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4200{
4201 u32 tmp;
4202
4203 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4204 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4205 drm_dbg_kms(&dev_priv->drm,
4206 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4207 tmp);
4208}
4209
4210static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4211{
4212 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4213
4214 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4215
4216 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4217
4218 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4219 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4220 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4221 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4222
4223 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4224 * gating disable must be set. Failure to set it results in
4225 * flickering pixels due to Z write ordering failures after
4226 * some amount of runtime in the Mesa "fire" demo, and Unigine
4227 * Sanctuary and Tropics, and apparently anything else with
4228 * alpha test or pixel discard.
4229 *
4230 * According to the spec, bit 11 (RCCUNIT) must also be set,
4231 * but we didn't debug actual testcases to find it out.
4232 *
4233 * WaDisableRCCUnitClockGating:snb
4234 * WaDisableRCPBUnitClockGating:snb
4235 */
4236 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4237 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4238 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4239
4240 /*
4241 * According to the spec the following bits should be
4242 * set in order to enable memory self-refresh and fbc:
4243 * The bit21 and bit22 of 0x42000
4244 * The bit21 and bit22 of 0x42004
4245 * The bit5 and bit7 of 0x42020
4246 * The bit14 of 0x70180
4247 * The bit14 of 0x71180
4248 *
4249 * WaFbcAsynchFlipDisableFbcQueue:snb
4250 */
4251 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4252 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4253 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4254 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4255 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4256 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4257 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4258 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4259 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4260 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4261
4262 g4x_disable_trickle_feed(dev_priv);
4263
4264 cpt_init_clock_gating(dev_priv);
4265
4266 gen6_check_mch_setup(dev_priv);
4267}
4268
4269static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4270{
4271 /*
4272 * TODO: this bit should only be enabled when really needed, then
4273 * disabled when not needed anymore in order to save power.
4274 */
4275 if (HAS_PCH_LPT_LP(dev_priv))
4276 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4277 0, PCH_LP_PARTITION_LEVEL_DISABLE);
4278
4279 /* WADPOClockGatingDisable:hsw */
4280 intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4281 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4282}
4283
4284static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4285{
4286 if (HAS_PCH_LPT_LP(dev_priv)) {
4287 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4288
4289 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4290 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4291 }
4292}
4293
4294static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4295 int general_prio_credits,
4296 int high_prio_credits)
4297{
4298 u32 misccpctl;
4299 u32 val;
4300
4301 /* WaTempDisableDOPClkGating:bdw */
4302 misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4303 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4304
4305 val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4306 val &= ~L3_PRIO_CREDITS_MASK;
4307 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4308 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4309 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4310
4311 /*
4312 * Wait at least 100 clocks before re-enabling clock gating.
4313 * See the definition of L3SQCREG1 in BSpec.
4314 */
4315 intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4316 udelay(1);
4317 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
4318}
4319
4320static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4321{
4322 /* Wa_1409120013:icl,ehl */
4323 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4324 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4325
4326 /*Wa_14010594013:icl, ehl */
4327 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4328 0, ICL_DELAY_PMRSP);
4329}
4330
4331static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4332{
4333 /* Wa_1409120013 */
4334 if (DISPLAY_VER(dev_priv) == 12)
4335 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4336 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4337
4338 /* Wa_1409825376:tgl (pre-prod)*/
4339 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4340 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4341
4342 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4343 if (DISPLAY_VER(dev_priv) == 12)
4344 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4345 CLKREQ_POLICY_MEM_UP_OVRD, 0);
4346}
4347
4348static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4349{
4350 gen12lp_init_clock_gating(dev_priv);
4351
4352 /* Wa_22011091694:adlp */
4353 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4354
4355 /* Bspec/49189 Initialize Sequence */
4356 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4357}
4358
4359static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4360{
4361 gen12lp_init_clock_gating(dev_priv);
4362
4363 /* Wa_1409836686:dg1[a0] */
4364 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4365 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4366}
4367
4368static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4369{
4370 /* Wa_22010146351:xehpsdv */
4371 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4372 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4373}
4374
4375static void dg2_init_clock_gating(struct drm_i915_private *i915)
4376{
4377 /* Wa_22010954014:dg2 */
4378 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4379 SGSI_SIDECLK_DIS);
4380
4381 /*
4382 * Wa_14010733611:dg2_g10
4383 * Wa_22010146351:dg2_g10
4384 */
4385 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4386 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4387 SGR_DIS | SGGI_DIS);
4388}
4389
4390static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4391{
4392 /* Wa_14012385139:pvc */
4393 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4394 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4395
4396 /* Wa_22010954014:pvc */
4397 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4398 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4399}
4400
4401static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4402{
4403 if (!HAS_PCH_CNP(dev_priv))
4404 return;
4405
4406 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4407 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4408}
4409
4410static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4411{
4412 cnp_init_clock_gating(dev_priv);
4413 gen9_init_clock_gating(dev_priv);
4414
4415 /* WAC6entrylatency:cfl */
4416 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4417
4418 /*
4419 * WaFbcTurnOffFbcWatermark:cfl
4420 * Display WA #0562: cfl
4421 */
4422 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4423
4424 /*
4425 * WaFbcNukeOnHostModify:cfl
4426 * Display WA #0873: cfl
4427 */
4428 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4429 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4430}
4431
4432static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4433{
4434 gen9_init_clock_gating(dev_priv);
4435
4436 /* WAC6entrylatency:kbl */
4437 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4438
4439 /* WaDisableSDEUnitClockGating:kbl */
4440 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4441 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4442 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4443
4444 /* WaDisableGamClockGating:kbl */
4445 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4446 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4447 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4448
4449 /*
4450 * WaFbcTurnOffFbcWatermark:kbl
4451 * Display WA #0562: kbl
4452 */
4453 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4454
4455 /*
4456 * WaFbcNukeOnHostModify:kbl
4457 * Display WA #0873: kbl
4458 */
4459 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4460 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4461}
4462
4463static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4464{
4465 gen9_init_clock_gating(dev_priv);
4466
4467 /* WaDisableDopClockGating:skl */
4468 intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4469 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4470
4471 /* WAC6entrylatency:skl */
4472 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4473
4474 /*
4475 * WaFbcTurnOffFbcWatermark:skl
4476 * Display WA #0562: skl
4477 */
4478 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4479
4480 /*
4481 * WaFbcNukeOnHostModify:skl
4482 * Display WA #0873: skl
4483 */
4484 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4485 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4486
4487 /*
4488 * WaFbcHighMemBwCorruptionAvoidance:skl
4489 * Display WA #0883: skl
4490 */
4491 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4492}
4493
4494static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4495{
4496 enum pipe pipe;
4497
4498 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4499 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4500
4501 /* WaSwitchSolVfFArbitrationPriority:bdw */
4502 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4503
4504 /* WaPsrDPAMaskVBlankInSRD:bdw */
4505 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4506
4507 for_each_pipe(dev_priv, pipe) {
4508 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4509 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4510 0, BDW_DPRS_MASK_VBLANK_SRD);
4511 }
4512
4513 /* WaVSRefCountFullforceMissDisable:bdw */
4514 /* WaDSRefCountFullforceMissDisable:bdw */
4515 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4516 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4517
4518 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4519 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4520
4521 /* WaDisableSDEUnitClockGating:bdw */
4522 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4523
4524 /* WaProgramL3SqcReg1Default:bdw */
4525 gen8_set_l3sqc_credits(dev_priv, 30, 2);
4526
4527 /* WaKVMNotificationOnConfigChange:bdw */
4528 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4529 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4530
4531 lpt_init_clock_gating(dev_priv);
4532
4533 /* WaDisableDopClockGating:bdw
4534 *
4535 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4536 * clock gating.
4537 */
4538 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4539}
4540
4541static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4542{
4543 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4544 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4545
4546 /* This is required by WaCatErrorRejectionIssue:hsw */
4547 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4548 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4549
4550 /* WaSwitchSolVfFArbitrationPriority:hsw */
4551 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4552
4553 lpt_init_clock_gating(dev_priv);
4554}
4555
4556static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4557{
4558 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4559
4560 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4561 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4562
4563 /* WaDisableBackToBackFlipFix:ivb */
4564 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4565 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4566 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4567
4568 if (IS_IVB_GT1(dev_priv))
4569 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4570 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4571 else {
4572 /* must write both registers */
4573 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4574 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4575 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4576 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4577 }
4578
4579 /*
4580 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4581 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4582 */
4583 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4584 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4585
4586 /* This is required by WaCatErrorRejectionIssue:ivb */
4587 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4588 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4589
4590 g4x_disable_trickle_feed(dev_priv);
4591
4592 intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4593 GEN6_MBC_SNPCR_MED);
4594
4595 if (!HAS_PCH_NOP(dev_priv))
4596 cpt_init_clock_gating(dev_priv);
4597
4598 gen6_check_mch_setup(dev_priv);
4599}
4600
4601static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4602{
4603 /* WaDisableBackToBackFlipFix:vlv */
4604 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4605 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4606 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4607
4608 /* WaDisableDopClockGating:vlv */
4609 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4610 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4611
4612 /* This is required by WaCatErrorRejectionIssue:vlv */
4613 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4614 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4615
4616 /*
4617 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4618 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4619 */
4620 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4621 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4622
4623 /* WaDisableL3Bank2xClockGate:vlv
4624 * Disabling L3 clock gating- MMIO 940c[25] = 1
4625 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4626 intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4627
4628 /*
4629 * WaDisableVLVClockGating_VBIIssue:vlv
4630 * Disable clock gating on th GCFG unit to prevent a delay
4631 * in the reporting of vblank events.
4632 */
4633 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4634}
4635
4636static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4637{
4638 /* WaVSRefCountFullforceMissDisable:chv */
4639 /* WaDSRefCountFullforceMissDisable:chv */
4640 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4641 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4642
4643 /* WaDisableSemaphoreAndSyncFlipWait:chv */
4644 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4645 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4646
4647 /* WaDisableCSUnitClockGating:chv */
4648 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4649
4650 /* WaDisableSDEUnitClockGating:chv */
4651 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4652
4653 /*
4654 * WaProgramL3SqcReg1Default:chv
4655 * See gfxspecs/Related Documents/Performance Guide/
4656 * LSQC Setting Recommendations.
4657 */
4658 gen8_set_l3sqc_credits(dev_priv, 38, 2);
4659}
4660
4661static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4662{
4663 u32 dspclk_gate;
4664
4665 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4666 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4667 GS_UNIT_CLOCK_GATE_DISABLE |
4668 CL_UNIT_CLOCK_GATE_DISABLE);
4669 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4670 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4671 OVRUNIT_CLOCK_GATE_DISABLE |
4672 OVCUNIT_CLOCK_GATE_DISABLE;
4673 if (IS_GM45(dev_priv))
4674 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4675 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4676
4677 g4x_disable_trickle_feed(dev_priv);
4678}
4679
4680static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4681{
4682 struct intel_uncore *uncore = &dev_priv->uncore;
4683
4684 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4685 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4686 intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4687 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4688 intel_uncore_write16(uncore, DEUC, 0);
4689 intel_uncore_write(uncore,
4690 MI_ARB_STATE,
4691 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4692}
4693
4694static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4695{
4696 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4697 I965_RCC_CLOCK_GATE_DISABLE |
4698 I965_RCPB_CLOCK_GATE_DISABLE |
4699 I965_ISC_CLOCK_GATE_DISABLE |
4700 I965_FBC_CLOCK_GATE_DISABLE);
4701 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4702 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4703 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4704}
4705
4706static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4707{
4708 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4709
4710 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4711 DSTATE_DOT_CLOCK_GATING;
4712 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4713
4714 if (IS_PINEVIEW(dev_priv))
4715 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4716 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4717
4718 /* IIR "flip pending" means done if this bit is set */
4719 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4720 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4721
4722 /* interrupts should cause a wake up from C3 */
4723 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4724
4725 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4726 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4727
4728 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4729 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4730}
4731
4732static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4733{
4734 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4735
4736 /* interrupts should cause a wake up from C3 */
4737 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4738 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4739
4740 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4741 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4742
4743 /*
4744 * Have FBC ignore 3D activity since we use software
4745 * render tracking, and otherwise a pure 3D workload
4746 * (even if it just renders a single frame and then does
4747 * abosultely nothing) would not allow FBC to recompress
4748 * until a 2D blit occurs.
4749 */
4750 intel_uncore_write(&dev_priv->uncore, SCPD0,
4751 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4752}
4753
4754static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4755{
4756 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4757 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4758 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4759}
4760
4761void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4762{
4763 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4764}
4765
4766void intel_suspend_hw(struct drm_i915_private *dev_priv)
4767{
4768 if (HAS_PCH_LPT(dev_priv))
4769 lpt_suspend_hw(dev_priv);
4770}
4771
4772static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4773{
4774 drm_dbg_kms(&dev_priv->drm,
4775 "No clock gating settings or workarounds applied.\n");
4776}
4777
4778#define CG_FUNCS(platform) \
4779static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4780 .init_clock_gating = platform##_init_clock_gating, \
4781}
4782
4783CG_FUNCS(pvc);
4784CG_FUNCS(dg2);
4785CG_FUNCS(xehpsdv);
4786CG_FUNCS(adlp);
4787CG_FUNCS(dg1);
4788CG_FUNCS(gen12lp);
4789CG_FUNCS(icl);
4790CG_FUNCS(cfl);
4791CG_FUNCS(skl);
4792CG_FUNCS(kbl);
4793CG_FUNCS(bxt);
4794CG_FUNCS(glk);
4795CG_FUNCS(bdw);
4796CG_FUNCS(chv);
4797CG_FUNCS(hsw);
4798CG_FUNCS(ivb);
4799CG_FUNCS(vlv);
4800CG_FUNCS(gen6);
4801CG_FUNCS(ilk);
4802CG_FUNCS(g4x);
4803CG_FUNCS(i965gm);
4804CG_FUNCS(i965g);
4805CG_FUNCS(gen3);
4806CG_FUNCS(i85x);
4807CG_FUNCS(i830);
4808CG_FUNCS(nop);
4809#undef CG_FUNCS
4810
4811/**
4812 * intel_init_clock_gating_hooks - setup the clock gating hooks
4813 * @dev_priv: device private
4814 *
4815 * Setup the hooks that configure which clocks of a given platform can be
4816 * gated and also apply various GT and display specific workarounds for these
4817 * platforms. Note that some GT specific workarounds are applied separately
4818 * when GPU contexts or batchbuffers start their execution.
4819 */
4820void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4821{
4822 if (IS_PONTEVECCHIO(dev_priv))
4823 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4824 else if (IS_DG2(dev_priv))
4825 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4826 else if (IS_XEHPSDV(dev_priv))
4827 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4828 else if (IS_ALDERLAKE_P(dev_priv))
4829 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4830 else if (IS_DG1(dev_priv))
4831 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4832 else if (GRAPHICS_VER(dev_priv) == 12)
4833 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4834 else if (GRAPHICS_VER(dev_priv) == 11)
4835 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4836 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4837 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4838 else if (IS_SKYLAKE(dev_priv))
4839 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4840 else if (IS_KABYLAKE(dev_priv))
4841 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4842 else if (IS_BROXTON(dev_priv))
4843 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4844 else if (IS_GEMINILAKE(dev_priv))
4845 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4846 else if (IS_BROADWELL(dev_priv))
4847 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4848 else if (IS_CHERRYVIEW(dev_priv))
4849 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4850 else if (IS_HASWELL(dev_priv))
4851 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4852 else if (IS_IVYBRIDGE(dev_priv))
4853 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4854 else if (IS_VALLEYVIEW(dev_priv))
4855 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4856 else if (GRAPHICS_VER(dev_priv) == 6)
4857 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4858 else if (GRAPHICS_VER(dev_priv) == 5)
4859 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4860 else if (IS_G4X(dev_priv))
4861 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4862 else if (IS_I965GM(dev_priv))
4863 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4864 else if (IS_I965G(dev_priv))
4865 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4866 else if (GRAPHICS_VER(dev_priv) == 3)
4867 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4868 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4869 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4870 else if (GRAPHICS_VER(dev_priv) == 2)
4871 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4872 else {
4873 MISSING_CASE(INTEL_DEVID(dev_priv));
4874 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4875 }
4876}
4877
4878static const struct intel_wm_funcs ilk_wm_funcs = {
4879 .compute_pipe_wm = ilk_compute_pipe_wm,
4880 .compute_intermediate_wm = ilk_compute_intermediate_wm,
4881 .initial_watermarks = ilk_initial_watermarks,
4882 .optimize_watermarks = ilk_optimize_watermarks,
4883};
4884
4885static const struct intel_wm_funcs vlv_wm_funcs = {
4886 .compute_pipe_wm = vlv_compute_pipe_wm,
4887 .compute_intermediate_wm = vlv_compute_intermediate_wm,
4888 .initial_watermarks = vlv_initial_watermarks,
4889 .optimize_watermarks = vlv_optimize_watermarks,
4890 .atomic_update_watermarks = vlv_atomic_update_fifo,
4891};
4892
4893static const struct intel_wm_funcs g4x_wm_funcs = {
4894 .compute_pipe_wm = g4x_compute_pipe_wm,
4895 .compute_intermediate_wm = g4x_compute_intermediate_wm,
4896 .initial_watermarks = g4x_initial_watermarks,
4897 .optimize_watermarks = g4x_optimize_watermarks,
4898};
4899
4900static const struct intel_wm_funcs pnv_wm_funcs = {
4901 .update_wm = pnv_update_wm,
4902};
4903
4904static const struct intel_wm_funcs i965_wm_funcs = {
4905 .update_wm = i965_update_wm,
4906};
4907
4908static const struct intel_wm_funcs i9xx_wm_funcs = {
4909 .update_wm = i9xx_update_wm,
4910};
4911
4912static const struct intel_wm_funcs i845_wm_funcs = {
4913 .update_wm = i845_update_wm,
4914};
4915
4916static const struct intel_wm_funcs nop_funcs = {
4917};
4918
4919/* Set up chip specific power management-related functions */
4920void intel_init_pm(struct drm_i915_private *dev_priv)
4921{
4922 if (DISPLAY_VER(dev_priv) >= 9) {
4923 skl_wm_init(dev_priv);
4924 return;
4925 }
4926
4927 /* For cxsr */
4928 if (IS_PINEVIEW(dev_priv))
4929 pnv_get_mem_freq(dev_priv);
4930 else if (GRAPHICS_VER(dev_priv) == 5)
4931 ilk_get_mem_freq(dev_priv);
4932
4933 /* For FIFO watermark updates */
4934 if (HAS_PCH_SPLIT(dev_priv)) {
4935 ilk_setup_wm_latency(dev_priv);
4936
4937 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4938 dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4939 (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4940 dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4941 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4942 } else {
4943 drm_dbg_kms(&dev_priv->drm,
4944 "Failed to read display plane latency. "
4945 "Disable CxSR\n");
4946 dev_priv->display.funcs.wm = &nop_funcs;
4947 }
4948 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4949 vlv_setup_wm_latency(dev_priv);
4950 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4951 } else if (IS_G4X(dev_priv)) {
4952 g4x_setup_wm_latency(dev_priv);
4953 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4954 } else if (IS_PINEVIEW(dev_priv)) {
4955 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4956 dev_priv->is_ddr3,
4957 dev_priv->fsb_freq,
4958 dev_priv->mem_freq)) {
4959 drm_info(&dev_priv->drm,
4960 "failed to find known CxSR latency "
4961 "(found ddr%s fsb freq %d, mem freq %d), "
4962 "disabling CxSR\n",
4963 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4964 dev_priv->fsb_freq, dev_priv->mem_freq);
4965 /* Disable CxSR and never update its watermark again */
4966 intel_set_memory_cxsr(dev_priv, false);
4967 dev_priv->display.funcs.wm = &nop_funcs;
4968 } else
4969 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4970 } else if (DISPLAY_VER(dev_priv) == 4) {
4971 dev_priv->display.funcs.wm = &i965_wm_funcs;
4972 } else if (DISPLAY_VER(dev_priv) == 3) {
4973 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4974 } else if (DISPLAY_VER(dev_priv) == 2) {
4975 if (INTEL_NUM_PIPES(dev_priv) == 1)
4976 dev_priv->display.funcs.wm = &i845_wm_funcs;
4977 else
4978 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4979 } else {
4980 drm_err(&dev_priv->drm,
4981 "unexpected fall-through in %s\n", __func__);
4982 dev_priv->display.funcs.wm = &nop_funcs;
4983 }
4984}
4985
4986void intel_pm_setup(struct drm_i915_private *dev_priv)
4987{
4988 dev_priv->runtime_pm.suspended = false;
4989 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
4990}