Loading...
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include "display/intel_de.h"
29#include "display/intel_display_trace.h"
30#include "display/skl_watermark.h"
31
32#include "gt/intel_engine_regs.h"
33#include "gt/intel_gt.h"
34#include "gt/intel_gt_mcr.h"
35#include "gt/intel_gt_regs.h"
36
37#include "i915_drv.h"
38#include "intel_mchbar_regs.h"
39#include "intel_pm.h"
40#include "vlv_sideband.h"
41
42struct drm_i915_clock_gating_funcs {
43 void (*init_clock_gating)(struct drm_i915_private *i915);
44};
45
46/* used in computing the new watermarks state */
47struct intel_wm_config {
48 unsigned int num_pipes_active;
49 bool sprites_enabled;
50 bool sprites_scaled;
51};
52
53static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
54{
55 if (HAS_LLC(dev_priv)) {
56 /*
57 * WaCompressedResourceDisplayNewHashMode:skl,kbl
58 * Display WA #0390: skl,kbl
59 *
60 * Must match Sampler, Pixel Back End, and Media. See
61 * WaCompressedResourceSamplerPbeMediaNewHashMode.
62 */
63 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
64 }
65
66 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
67 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
68
69 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
70 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
71
72 /*
73 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
74 * Display WA #0859: skl,bxt,kbl,glk,cfl
75 */
76 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
77}
78
79static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
80{
81 gen9_init_clock_gating(dev_priv);
82
83 /* WaDisableSDEUnitClockGating:bxt */
84 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
85
86 /*
87 * FIXME:
88 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
89 */
90 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
91
92 /*
93 * Wa: Backlight PWM may stop in the asserted state, causing backlight
94 * to stay fully on.
95 */
96 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
97 PWM1_GATING_DIS | PWM2_GATING_DIS);
98
99 /*
100 * Lower the display internal timeout.
101 * This is needed to avoid any hard hangs when DSI port PLL
102 * is off and a MMIO access is attempted by any privilege
103 * application, using batch buffers or any other means.
104 */
105 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
106
107 /*
108 * WaFbcTurnOffFbcWatermark:bxt
109 * Display WA #0562: bxt
110 */
111 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
112
113 /*
114 * WaFbcHighMemBwCorruptionAvoidance:bxt
115 * Display WA #0883: bxt
116 */
117 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
118}
119
120static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124 /*
125 * WaDisablePWMClockGating:glk
126 * Backlight PWM may stop in the asserted state, causing backlight
127 * to stay fully on.
128 */
129 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
130 PWM1_GATING_DIS | PWM2_GATING_DIS);
131}
132
133static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
134{
135 u32 tmp;
136
137 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
138
139 switch (tmp & CLKCFG_FSB_MASK) {
140 case CLKCFG_FSB_533:
141 dev_priv->fsb_freq = 533; /* 133*4 */
142 break;
143 case CLKCFG_FSB_800:
144 dev_priv->fsb_freq = 800; /* 200*4 */
145 break;
146 case CLKCFG_FSB_667:
147 dev_priv->fsb_freq = 667; /* 167*4 */
148 break;
149 case CLKCFG_FSB_400:
150 dev_priv->fsb_freq = 400; /* 100*4 */
151 break;
152 }
153
154 switch (tmp & CLKCFG_MEM_MASK) {
155 case CLKCFG_MEM_533:
156 dev_priv->mem_freq = 533;
157 break;
158 case CLKCFG_MEM_667:
159 dev_priv->mem_freq = 667;
160 break;
161 case CLKCFG_MEM_800:
162 dev_priv->mem_freq = 800;
163 break;
164 }
165
166 /* detect pineview DDR3 setting */
167 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
168 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
169}
170
171static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
172{
173 u16 ddrpll, csipll;
174
175 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
176 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
177
178 switch (ddrpll & 0xff) {
179 case 0xc:
180 dev_priv->mem_freq = 800;
181 break;
182 case 0x10:
183 dev_priv->mem_freq = 1066;
184 break;
185 case 0x14:
186 dev_priv->mem_freq = 1333;
187 break;
188 case 0x18:
189 dev_priv->mem_freq = 1600;
190 break;
191 default:
192 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
193 ddrpll & 0xff);
194 dev_priv->mem_freq = 0;
195 break;
196 }
197
198 switch (csipll & 0x3ff) {
199 case 0x00c:
200 dev_priv->fsb_freq = 3200;
201 break;
202 case 0x00e:
203 dev_priv->fsb_freq = 3733;
204 break;
205 case 0x010:
206 dev_priv->fsb_freq = 4266;
207 break;
208 case 0x012:
209 dev_priv->fsb_freq = 4800;
210 break;
211 case 0x014:
212 dev_priv->fsb_freq = 5333;
213 break;
214 case 0x016:
215 dev_priv->fsb_freq = 5866;
216 break;
217 case 0x018:
218 dev_priv->fsb_freq = 6400;
219 break;
220 default:
221 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
222 csipll & 0x3ff);
223 dev_priv->fsb_freq = 0;
224 break;
225 }
226}
227
228static const struct cxsr_latency cxsr_latency_table[] = {
229 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
230 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
231 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
232 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
233 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
234
235 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
236 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
237 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
238 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
239 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
240
241 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
242 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
243 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
244 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
245 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
246
247 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
248 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
249 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
250 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
251 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
252
253 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
254 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
255 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
256 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
257 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
258
259 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
260 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
261 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
262 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
263 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
264};
265
266static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
267 bool is_ddr3,
268 int fsb,
269 int mem)
270{
271 const struct cxsr_latency *latency;
272 int i;
273
274 if (fsb == 0 || mem == 0)
275 return NULL;
276
277 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
278 latency = &cxsr_latency_table[i];
279 if (is_desktop == latency->is_desktop &&
280 is_ddr3 == latency->is_ddr3 &&
281 fsb == latency->fsb_freq && mem == latency->mem_freq)
282 return latency;
283 }
284
285 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
286
287 return NULL;
288}
289
290static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
291{
292 u32 val;
293
294 vlv_punit_get(dev_priv);
295
296 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
297 if (enable)
298 val &= ~FORCE_DDR_HIGH_FREQ;
299 else
300 val |= FORCE_DDR_HIGH_FREQ;
301 val &= ~FORCE_DDR_LOW_FREQ;
302 val |= FORCE_DDR_FREQ_REQ_ACK;
303 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
304
305 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
306 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
307 drm_err(&dev_priv->drm,
308 "timed out waiting for Punit DDR DVFS request\n");
309
310 vlv_punit_put(dev_priv);
311}
312
313static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
314{
315 u32 val;
316
317 vlv_punit_get(dev_priv);
318
319 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
320 if (enable)
321 val |= DSP_MAXFIFO_PM5_ENABLE;
322 else
323 val &= ~DSP_MAXFIFO_PM5_ENABLE;
324 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
325
326 vlv_punit_put(dev_priv);
327}
328
329#define FW_WM(value, plane) \
330 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
331
332static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
333{
334 bool was_enabled;
335 u32 val;
336
337 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
338 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
339 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
340 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
341 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
342 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
343 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
344 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
345 } else if (IS_PINEVIEW(dev_priv)) {
346 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
347 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
348 if (enable)
349 val |= PINEVIEW_SELF_REFRESH_EN;
350 else
351 val &= ~PINEVIEW_SELF_REFRESH_EN;
352 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
353 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
354 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
355 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
356 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
357 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
358 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
359 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
360 } else if (IS_I915GM(dev_priv)) {
361 /*
362 * FIXME can't find a bit like this for 915G, and
363 * and yet it does have the related watermark in
364 * FW_BLC_SELF. What's going on?
365 */
366 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
367 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
368 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
369 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
370 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
371 } else {
372 return false;
373 }
374
375 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
376
377 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
378 str_enabled_disabled(enable),
379 str_enabled_disabled(was_enabled));
380
381 return was_enabled;
382}
383
384/**
385 * intel_set_memory_cxsr - Configure CxSR state
386 * @dev_priv: i915 device
387 * @enable: Allow vs. disallow CxSR
388 *
389 * Allow or disallow the system to enter a special CxSR
390 * (C-state self refresh) state. What typically happens in CxSR mode
391 * is that several display FIFOs may get combined into a single larger
392 * FIFO for a particular plane (so called max FIFO mode) to allow the
393 * system to defer memory fetches longer, and the memory will enter
394 * self refresh.
395 *
396 * Note that enabling CxSR does not guarantee that the system enter
397 * this special mode, nor does it guarantee that the system stays
398 * in that mode once entered. So this just allows/disallows the system
399 * to autonomously utilize the CxSR mode. Other factors such as core
400 * C-states will affect when/if the system actually enters/exits the
401 * CxSR mode.
402 *
403 * Note that on VLV/CHV this actually only controls the max FIFO mode,
404 * and the system is free to enter/exit memory self refresh at any time
405 * even when the use of CxSR has been disallowed.
406 *
407 * While the system is actually in the CxSR/max FIFO mode, some plane
408 * control registers will not get latched on vblank. Thus in order to
409 * guarantee the system will respond to changes in the plane registers
410 * we must always disallow CxSR prior to making changes to those registers.
411 * Unfortunately the system will re-evaluate the CxSR conditions at
412 * frame start which happens after vblank start (which is when the plane
413 * registers would get latched), so we can't proceed with the plane update
414 * during the same frame where we disallowed CxSR.
415 *
416 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
417 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
418 * the hardware w.r.t. HPLL SR when writing to plane registers.
419 * Disallowing just CxSR is sufficient.
420 */
421bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
422{
423 bool ret;
424
425 mutex_lock(&dev_priv->display.wm.wm_mutex);
426 ret = _intel_set_memory_cxsr(dev_priv, enable);
427 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
428 dev_priv->display.wm.vlv.cxsr = enable;
429 else if (IS_G4X(dev_priv))
430 dev_priv->display.wm.g4x.cxsr = enable;
431 mutex_unlock(&dev_priv->display.wm.wm_mutex);
432
433 return ret;
434}
435
436/*
437 * Latency for FIFO fetches is dependent on several factors:
438 * - memory configuration (speed, channels)
439 * - chipset
440 * - current MCH state
441 * It can be fairly high in some situations, so here we assume a fairly
442 * pessimal value. It's a tradeoff between extra memory fetches (if we
443 * set this value too high, the FIFO will fetch frequently to stay full)
444 * and power consumption (set it too low to save power and we might see
445 * FIFO underruns and display "flicker").
446 *
447 * A value of 5us seems to be a good balance; safe for very low end
448 * platforms but not overly aggressive on lower latency configs.
449 */
450static const int pessimal_latency_ns = 5000;
451
452#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
453 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
454
455static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
456{
457 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
459 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
460 enum pipe pipe = crtc->pipe;
461 int sprite0_start, sprite1_start;
462 u32 dsparb, dsparb2, dsparb3;
463
464 switch (pipe) {
465 case PIPE_A:
466 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
467 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
468 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
469 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
470 break;
471 case PIPE_B:
472 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
473 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
474 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
475 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
476 break;
477 case PIPE_C:
478 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
479 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
480 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
481 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
482 break;
483 default:
484 MISSING_CASE(pipe);
485 return;
486 }
487
488 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
489 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
490 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
491 fifo_state->plane[PLANE_CURSOR] = 63;
492}
493
494static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
495 enum i9xx_plane_id i9xx_plane)
496{
497 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
498 int size;
499
500 size = dsparb & 0x7f;
501 if (i9xx_plane == PLANE_B)
502 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
503
504 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
505 dsparb, plane_name(i9xx_plane), size);
506
507 return size;
508}
509
510static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
511 enum i9xx_plane_id i9xx_plane)
512{
513 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
514 int size;
515
516 size = dsparb & 0x1ff;
517 if (i9xx_plane == PLANE_B)
518 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
519 size >>= 1; /* Convert to cachelines */
520
521 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
522 dsparb, plane_name(i9xx_plane), size);
523
524 return size;
525}
526
527static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
528 enum i9xx_plane_id i9xx_plane)
529{
530 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
531 int size;
532
533 size = dsparb & 0x7f;
534 size >>= 2; /* Convert to cachelines */
535
536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
538
539 return size;
540}
541
542/* Pineview has different values for various configs */
543static const struct intel_watermark_params pnv_display_wm = {
544 .fifo_size = PINEVIEW_DISPLAY_FIFO,
545 .max_wm = PINEVIEW_MAX_WM,
546 .default_wm = PINEVIEW_DFT_WM,
547 .guard_size = PINEVIEW_GUARD_WM,
548 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
549};
550
551static const struct intel_watermark_params pnv_display_hplloff_wm = {
552 .fifo_size = PINEVIEW_DISPLAY_FIFO,
553 .max_wm = PINEVIEW_MAX_WM,
554 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
555 .guard_size = PINEVIEW_GUARD_WM,
556 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
557};
558
559static const struct intel_watermark_params pnv_cursor_wm = {
560 .fifo_size = PINEVIEW_CURSOR_FIFO,
561 .max_wm = PINEVIEW_CURSOR_MAX_WM,
562 .default_wm = PINEVIEW_CURSOR_DFT_WM,
563 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
564 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
565};
566
567static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
568 .fifo_size = PINEVIEW_CURSOR_FIFO,
569 .max_wm = PINEVIEW_CURSOR_MAX_WM,
570 .default_wm = PINEVIEW_CURSOR_DFT_WM,
571 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
572 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
573};
574
575static const struct intel_watermark_params i965_cursor_wm_info = {
576 .fifo_size = I965_CURSOR_FIFO,
577 .max_wm = I965_CURSOR_MAX_WM,
578 .default_wm = I965_CURSOR_DFT_WM,
579 .guard_size = 2,
580 .cacheline_size = I915_FIFO_LINE_SIZE,
581};
582
583static const struct intel_watermark_params i945_wm_info = {
584 .fifo_size = I945_FIFO_SIZE,
585 .max_wm = I915_MAX_WM,
586 .default_wm = 1,
587 .guard_size = 2,
588 .cacheline_size = I915_FIFO_LINE_SIZE,
589};
590
591static const struct intel_watermark_params i915_wm_info = {
592 .fifo_size = I915_FIFO_SIZE,
593 .max_wm = I915_MAX_WM,
594 .default_wm = 1,
595 .guard_size = 2,
596 .cacheline_size = I915_FIFO_LINE_SIZE,
597};
598
599static const struct intel_watermark_params i830_a_wm_info = {
600 .fifo_size = I855GM_FIFO_SIZE,
601 .max_wm = I915_MAX_WM,
602 .default_wm = 1,
603 .guard_size = 2,
604 .cacheline_size = I830_FIFO_LINE_SIZE,
605};
606
607static const struct intel_watermark_params i830_bc_wm_info = {
608 .fifo_size = I855GM_FIFO_SIZE,
609 .max_wm = I915_MAX_WM/2,
610 .default_wm = 1,
611 .guard_size = 2,
612 .cacheline_size = I830_FIFO_LINE_SIZE,
613};
614
615static const struct intel_watermark_params i845_wm_info = {
616 .fifo_size = I830_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
618 .default_wm = 1,
619 .guard_size = 2,
620 .cacheline_size = I830_FIFO_LINE_SIZE,
621};
622
623/**
624 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
625 * @pixel_rate: Pipe pixel rate in kHz
626 * @cpp: Plane bytes per pixel
627 * @latency: Memory wakeup latency in 0.1us units
628 *
629 * Compute the watermark using the method 1 or "small buffer"
630 * formula. The caller may additonally add extra cachelines
631 * to account for TLB misses and clock crossings.
632 *
633 * This method is concerned with the short term drain rate
634 * of the FIFO, ie. it does not account for blanking periods
635 * which would effectively reduce the average drain rate across
636 * a longer period. The name "small" refers to the fact the
637 * FIFO is relatively small compared to the amount of data
638 * fetched.
639 *
640 * The FIFO level vs. time graph might look something like:
641 *
642 * |\ |\
643 * | \ | \
644 * __---__---__ (- plane active, _ blanking)
645 * -> time
646 *
647 * or perhaps like this:
648 *
649 * |\|\ |\|\
650 * __----__----__ (- plane active, _ blanking)
651 * -> time
652 *
653 * Returns:
654 * The watermark in bytes
655 */
656static unsigned int intel_wm_method1(unsigned int pixel_rate,
657 unsigned int cpp,
658 unsigned int latency)
659{
660 u64 ret;
661
662 ret = mul_u32_u32(pixel_rate, cpp * latency);
663 ret = DIV_ROUND_UP_ULL(ret, 10000);
664
665 return ret;
666}
667
668/**
669 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @htotal: Pipe horizontal total
672 * @width: Plane width in pixels
673 * @cpp: Plane bytes per pixel
674 * @latency: Memory wakeup latency in 0.1us units
675 *
676 * Compute the watermark using the method 2 or "large buffer"
677 * formula. The caller may additonally add extra cachelines
678 * to account for TLB misses and clock crossings.
679 *
680 * This method is concerned with the long term drain rate
681 * of the FIFO, ie. it does account for blanking periods
682 * which effectively reduce the average drain rate across
683 * a longer period. The name "large" refers to the fact the
684 * FIFO is relatively large compared to the amount of data
685 * fetched.
686 *
687 * The FIFO level vs. time graph might look something like:
688 *
689 * |\___ |\___
690 * | \___ | \___
691 * | \ | \
692 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
693 * -> time
694 *
695 * Returns:
696 * The watermark in bytes
697 */
698static unsigned int intel_wm_method2(unsigned int pixel_rate,
699 unsigned int htotal,
700 unsigned int width,
701 unsigned int cpp,
702 unsigned int latency)
703{
704 unsigned int ret;
705
706 /*
707 * FIXME remove once all users are computing
708 * watermarks in the correct place.
709 */
710 if (WARN_ON_ONCE(htotal == 0))
711 htotal = 1;
712
713 ret = (latency * pixel_rate) / (htotal * 10000);
714 ret = (ret + 1) * width * cpp;
715
716 return ret;
717}
718
719/**
720 * intel_calculate_wm - calculate watermark level
721 * @pixel_rate: pixel clock
722 * @wm: chip FIFO params
723 * @fifo_size: size of the FIFO buffer
724 * @cpp: bytes per pixel
725 * @latency_ns: memory latency for the platform
726 *
727 * Calculate the watermark level (the level at which the display plane will
728 * start fetching from memory again). Each chip has a different display
729 * FIFO size and allocation, so the caller needs to figure that out and pass
730 * in the correct intel_watermark_params structure.
731 *
732 * As the pixel clock runs, the FIFO will be drained at a rate that depends
733 * on the pixel size. When it reaches the watermark level, it'll start
734 * fetching FIFO line sized based chunks from memory until the FIFO fills
735 * past the watermark point. If the FIFO drains completely, a FIFO underrun
736 * will occur, and a display engine hang could result.
737 */
738static unsigned int intel_calculate_wm(int pixel_rate,
739 const struct intel_watermark_params *wm,
740 int fifo_size, int cpp,
741 unsigned int latency_ns)
742{
743 int entries, wm_size;
744
745 /*
746 * Note: we need to make sure we don't overflow for various clock &
747 * latency values.
748 * clocks go from a few thousand to several hundred thousand.
749 * latency is usually a few thousand
750 */
751 entries = intel_wm_method1(pixel_rate, cpp,
752 latency_ns / 100);
753 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
754 wm->guard_size;
755 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
756
757 wm_size = fifo_size - entries;
758 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
759
760 /* Don't promote wm_size to unsigned... */
761 if (wm_size > wm->max_wm)
762 wm_size = wm->max_wm;
763 if (wm_size <= 0)
764 wm_size = wm->default_wm;
765
766 /*
767 * Bspec seems to indicate that the value shouldn't be lower than
768 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
769 * Lets go for 8 which is the burst size since certain platforms
770 * already use a hardcoded 8 (which is what the spec says should be
771 * done).
772 */
773 if (wm_size <= 8)
774 wm_size = 8;
775
776 return wm_size;
777}
778
779static bool is_disabling(int old, int new, int threshold)
780{
781 return old >= threshold && new < threshold;
782}
783
784static bool is_enabling(int old, int new, int threshold)
785{
786 return old < threshold && new >= threshold;
787}
788
789static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
790{
791 return dev_priv->display.wm.max_level + 1;
792}
793
794bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
795 const struct intel_plane_state *plane_state)
796{
797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
798
799 /* FIXME check the 'enable' instead */
800 if (!crtc_state->hw.active)
801 return false;
802
803 /*
804 * Treat cursor with fb as always visible since cursor updates
805 * can happen faster than the vrefresh rate, and the current
806 * watermark code doesn't handle that correctly. Cursor updates
807 * which set/clear the fb or change the cursor size are going
808 * to get throttled by intel_legacy_cursor_update() to work
809 * around this problem with the watermark code.
810 */
811 if (plane->id == PLANE_CURSOR)
812 return plane_state->hw.fb != NULL;
813 else
814 return plane_state->uapi.visible;
815}
816
817static bool intel_crtc_active(struct intel_crtc *crtc)
818{
819 /* Be paranoid as we can arrive here with only partial
820 * state retrieved from the hardware during setup.
821 *
822 * We can ditch the adjusted_mode.crtc_clock check as soon
823 * as Haswell has gained clock readout/fastboot support.
824 *
825 * We can ditch the crtc->primary->state->fb check as soon as we can
826 * properly reconstruct framebuffers.
827 *
828 * FIXME: The intel_crtc->active here should be switched to
829 * crtc->state->active once we have proper CRTC states wired up
830 * for atomic.
831 */
832 return crtc && crtc->active && crtc->base.primary->state->fb &&
833 crtc->config->hw.adjusted_mode.crtc_clock;
834}
835
836static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
837{
838 struct intel_crtc *crtc, *enabled = NULL;
839
840 for_each_intel_crtc(&dev_priv->drm, crtc) {
841 if (intel_crtc_active(crtc)) {
842 if (enabled)
843 return NULL;
844 enabled = crtc;
845 }
846 }
847
848 return enabled;
849}
850
851static void pnv_update_wm(struct drm_i915_private *dev_priv)
852{
853 struct intel_crtc *crtc;
854 const struct cxsr_latency *latency;
855 u32 reg;
856 unsigned int wm;
857
858 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
859 dev_priv->is_ddr3,
860 dev_priv->fsb_freq,
861 dev_priv->mem_freq);
862 if (!latency) {
863 drm_dbg_kms(&dev_priv->drm,
864 "Unknown FSB/MEM found, disable CxSR\n");
865 intel_set_memory_cxsr(dev_priv, false);
866 return;
867 }
868
869 crtc = single_enabled_crtc(dev_priv);
870 if (crtc) {
871 const struct drm_framebuffer *fb =
872 crtc->base.primary->state->fb;
873 int pixel_rate = crtc->config->pixel_rate;
874 int cpp = fb->format->cpp[0];
875
876 /* Display SR */
877 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
878 pnv_display_wm.fifo_size,
879 cpp, latency->display_sr);
880 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
881 reg &= ~DSPFW_SR_MASK;
882 reg |= FW_WM(wm, SR);
883 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
884 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
885
886 /* cursor SR */
887 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
888 pnv_display_wm.fifo_size,
889 4, latency->cursor_sr);
890 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
891 FW_WM(wm, CURSOR_SR));
892
893 /* Display HPLL off SR */
894 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
895 pnv_display_hplloff_wm.fifo_size,
896 cpp, latency->display_hpll_disable);
897 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
898
899 /* cursor HPLL off SR */
900 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
901 pnv_display_hplloff_wm.fifo_size,
902 4, latency->cursor_hpll_disable);
903 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
904 reg &= ~DSPFW_HPLL_CURSOR_MASK;
905 reg |= FW_WM(wm, HPLL_CURSOR);
906 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
907 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
908
909 intel_set_memory_cxsr(dev_priv, true);
910 } else {
911 intel_set_memory_cxsr(dev_priv, false);
912 }
913}
914
915/*
916 * Documentation says:
917 * "If the line size is small, the TLB fetches can get in the way of the
918 * data fetches, causing some lag in the pixel data return which is not
919 * accounted for in the above formulas. The following adjustment only
920 * needs to be applied if eight whole lines fit in the buffer at once.
921 * The WM is adjusted upwards by the difference between the FIFO size
922 * and the size of 8 whole lines. This adjustment is always performed
923 * in the actual pixel depth regardless of whether FBC is enabled or not."
924 */
925static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
926{
927 int tlb_miss = fifo_size * 64 - width * cpp * 8;
928
929 return max(0, tlb_miss);
930}
931
932static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
933 const struct g4x_wm_values *wm)
934{
935 enum pipe pipe;
936
937 for_each_pipe(dev_priv, pipe)
938 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
939
940 intel_uncore_write(&dev_priv->uncore, DSPFW1,
941 FW_WM(wm->sr.plane, SR) |
942 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
943 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
944 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
945 intel_uncore_write(&dev_priv->uncore, DSPFW2,
946 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
947 FW_WM(wm->sr.fbc, FBC_SR) |
948 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
949 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
950 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
951 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
952 intel_uncore_write(&dev_priv->uncore, DSPFW3,
953 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
954 FW_WM(wm->sr.cursor, CURSOR_SR) |
955 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
956 FW_WM(wm->hpll.plane, HPLL_SR));
957
958 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
959}
960
961#define FW_WM_VLV(value, plane) \
962 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
963
964static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
965 const struct vlv_wm_values *wm)
966{
967 enum pipe pipe;
968
969 for_each_pipe(dev_priv, pipe) {
970 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
971
972 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
973 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
974 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
975 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
976 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
977 }
978
979 /*
980 * Zero the (unused) WM1 watermarks, and also clear all the
981 * high order bits so that there are no out of bounds values
982 * present in the registers during the reprogramming.
983 */
984 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
985 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
986 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
987 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
988 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
989
990 intel_uncore_write(&dev_priv->uncore, DSPFW1,
991 FW_WM(wm->sr.plane, SR) |
992 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
993 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
994 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
995 intel_uncore_write(&dev_priv->uncore, DSPFW2,
996 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
998 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
999 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1000 FW_WM(wm->sr.cursor, CURSOR_SR));
1001
1002 if (IS_CHERRYVIEW(dev_priv)) {
1003 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1004 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1005 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1006 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1007 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1008 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1009 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1010 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1011 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1012 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1013 FW_WM(wm->sr.plane >> 9, SR_HI) |
1014 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1015 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1016 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1017 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1018 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1019 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1020 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1021 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1022 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1023 } else {
1024 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1025 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1026 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1027 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1028 FW_WM(wm->sr.plane >> 9, SR_HI) |
1029 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1030 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1031 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1032 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1033 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1034 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1035 }
1036
1037 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1038}
1039
1040#undef FW_WM_VLV
1041
1042static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1043{
1044 /* all latencies in usec */
1045 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1046 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1047 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1048
1049 dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1050}
1051
1052static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1053{
1054 /*
1055 * DSPCNTR[13] supposedly controls whether the
1056 * primary plane can use the FIFO space otherwise
1057 * reserved for the sprite plane. It's not 100% clear
1058 * what the actual FIFO size is, but it looks like we
1059 * can happily set both primary and sprite watermarks
1060 * up to 127 cachelines. So that would seem to mean
1061 * that either DSPCNTR[13] doesn't do anything, or that
1062 * the total FIFO is >= 256 cachelines in size. Either
1063 * way, we don't seem to have to worry about this
1064 * repartitioning as the maximum watermark value the
1065 * register can hold for each plane is lower than the
1066 * minimum FIFO size.
1067 */
1068 switch (plane_id) {
1069 case PLANE_CURSOR:
1070 return 63;
1071 case PLANE_PRIMARY:
1072 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1073 case PLANE_SPRITE0:
1074 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1075 default:
1076 MISSING_CASE(plane_id);
1077 return 0;
1078 }
1079}
1080
1081static int g4x_fbc_fifo_size(int level)
1082{
1083 switch (level) {
1084 case G4X_WM_LEVEL_SR:
1085 return 7;
1086 case G4X_WM_LEVEL_HPLL:
1087 return 15;
1088 default:
1089 MISSING_CASE(level);
1090 return 0;
1091 }
1092}
1093
1094static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1095 const struct intel_plane_state *plane_state,
1096 int level)
1097{
1098 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1099 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1100 const struct drm_display_mode *pipe_mode =
1101 &crtc_state->hw.pipe_mode;
1102 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1103 unsigned int pixel_rate, htotal, cpp, width, wm;
1104
1105 if (latency == 0)
1106 return USHRT_MAX;
1107
1108 if (!intel_wm_plane_visible(crtc_state, plane_state))
1109 return 0;
1110
1111 cpp = plane_state->hw.fb->format->cpp[0];
1112
1113 /*
1114 * WaUse32BppForSRWM:ctg,elk
1115 *
1116 * The spec fails to list this restriction for the
1117 * HPLL watermark, which seems a little strange.
1118 * Let's use 32bpp for the HPLL watermark as well.
1119 */
1120 if (plane->id == PLANE_PRIMARY &&
1121 level != G4X_WM_LEVEL_NORMAL)
1122 cpp = max(cpp, 4u);
1123
1124 pixel_rate = crtc_state->pixel_rate;
1125 htotal = pipe_mode->crtc_htotal;
1126 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1127
1128 if (plane->id == PLANE_CURSOR) {
1129 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1130 } else if (plane->id == PLANE_PRIMARY &&
1131 level == G4X_WM_LEVEL_NORMAL) {
1132 wm = intel_wm_method1(pixel_rate, cpp, latency);
1133 } else {
1134 unsigned int small, large;
1135
1136 small = intel_wm_method1(pixel_rate, cpp, latency);
1137 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1138
1139 wm = min(small, large);
1140 }
1141
1142 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1143 width, cpp);
1144
1145 wm = DIV_ROUND_UP(wm, 64) + 2;
1146
1147 return min_t(unsigned int, wm, USHRT_MAX);
1148}
1149
1150static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1151 int level, enum plane_id plane_id, u16 value)
1152{
1153 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1154 bool dirty = false;
1155
1156 for (; level < intel_wm_num_levels(dev_priv); level++) {
1157 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1158
1159 dirty |= raw->plane[plane_id] != value;
1160 raw->plane[plane_id] = value;
1161 }
1162
1163 return dirty;
1164}
1165
1166static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1167 int level, u16 value)
1168{
1169 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1170 bool dirty = false;
1171
1172 /* NORMAL level doesn't have an FBC watermark */
1173 level = max(level, G4X_WM_LEVEL_SR);
1174
1175 for (; level < intel_wm_num_levels(dev_priv); level++) {
1176 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1177
1178 dirty |= raw->fbc != value;
1179 raw->fbc = value;
1180 }
1181
1182 return dirty;
1183}
1184
1185static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1186 const struct intel_plane_state *plane_state,
1187 u32 pri_val);
1188
1189static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1190 const struct intel_plane_state *plane_state)
1191{
1192 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1193 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1195 enum plane_id plane_id = plane->id;
1196 bool dirty = false;
1197 int level;
1198
1199 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1200 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1201 if (plane_id == PLANE_PRIMARY)
1202 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1203 goto out;
1204 }
1205
1206 for (level = 0; level < num_levels; level++) {
1207 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1208 int wm, max_wm;
1209
1210 wm = g4x_compute_wm(crtc_state, plane_state, level);
1211 max_wm = g4x_plane_fifo_size(plane_id, level);
1212
1213 if (wm > max_wm)
1214 break;
1215
1216 dirty |= raw->plane[plane_id] != wm;
1217 raw->plane[plane_id] = wm;
1218
1219 if (plane_id != PLANE_PRIMARY ||
1220 level == G4X_WM_LEVEL_NORMAL)
1221 continue;
1222
1223 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1224 raw->plane[plane_id]);
1225 max_wm = g4x_fbc_fifo_size(level);
1226
1227 /*
1228 * FBC wm is not mandatory as we
1229 * can always just disable its use.
1230 */
1231 if (wm > max_wm)
1232 wm = USHRT_MAX;
1233
1234 dirty |= raw->fbc != wm;
1235 raw->fbc = wm;
1236 }
1237
1238 /* mark watermarks as invalid */
1239 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1240
1241 if (plane_id == PLANE_PRIMARY)
1242 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1243
1244 out:
1245 if (dirty) {
1246 drm_dbg_kms(&dev_priv->drm,
1247 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1248 plane->base.name,
1249 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1250 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1251 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1252
1253 if (plane_id == PLANE_PRIMARY)
1254 drm_dbg_kms(&dev_priv->drm,
1255 "FBC watermarks: SR=%d, HPLL=%d\n",
1256 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1257 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1258 }
1259
1260 return dirty;
1261}
1262
1263static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1264 enum plane_id plane_id, int level)
1265{
1266 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1267
1268 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1269}
1270
1271static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1272 int level)
1273{
1274 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1275
1276 if (level > dev_priv->display.wm.max_level)
1277 return false;
1278
1279 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1280 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1281 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1282}
1283
1284/* mark all levels starting from 'level' as invalid */
1285static void g4x_invalidate_wms(struct intel_crtc *crtc,
1286 struct g4x_wm_state *wm_state, int level)
1287{
1288 if (level <= G4X_WM_LEVEL_NORMAL) {
1289 enum plane_id plane_id;
1290
1291 for_each_plane_id_on_crtc(crtc, plane_id)
1292 wm_state->wm.plane[plane_id] = USHRT_MAX;
1293 }
1294
1295 if (level <= G4X_WM_LEVEL_SR) {
1296 wm_state->cxsr = false;
1297 wm_state->sr.cursor = USHRT_MAX;
1298 wm_state->sr.plane = USHRT_MAX;
1299 wm_state->sr.fbc = USHRT_MAX;
1300 }
1301
1302 if (level <= G4X_WM_LEVEL_HPLL) {
1303 wm_state->hpll_en = false;
1304 wm_state->hpll.cursor = USHRT_MAX;
1305 wm_state->hpll.plane = USHRT_MAX;
1306 wm_state->hpll.fbc = USHRT_MAX;
1307 }
1308}
1309
1310static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1311 int level)
1312{
1313 if (level < G4X_WM_LEVEL_SR)
1314 return false;
1315
1316 if (level >= G4X_WM_LEVEL_SR &&
1317 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1318 return false;
1319
1320 if (level >= G4X_WM_LEVEL_HPLL &&
1321 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1322 return false;
1323
1324 return true;
1325}
1326
1327static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1328{
1329 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1330 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1331 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1332 const struct g4x_pipe_wm *raw;
1333 enum plane_id plane_id;
1334 int level;
1335
1336 level = G4X_WM_LEVEL_NORMAL;
1337 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1338 goto out;
1339
1340 raw = &crtc_state->wm.g4x.raw[level];
1341 for_each_plane_id_on_crtc(crtc, plane_id)
1342 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1343
1344 level = G4X_WM_LEVEL_SR;
1345 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1346 goto out;
1347
1348 raw = &crtc_state->wm.g4x.raw[level];
1349 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1350 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1351 wm_state->sr.fbc = raw->fbc;
1352
1353 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1354
1355 level = G4X_WM_LEVEL_HPLL;
1356 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1357 goto out;
1358
1359 raw = &crtc_state->wm.g4x.raw[level];
1360 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1361 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1362 wm_state->hpll.fbc = raw->fbc;
1363
1364 wm_state->hpll_en = wm_state->cxsr;
1365
1366 level++;
1367
1368 out:
1369 if (level == G4X_WM_LEVEL_NORMAL)
1370 return -EINVAL;
1371
1372 /* invalidate the higher levels */
1373 g4x_invalidate_wms(crtc, wm_state, level);
1374
1375 /*
1376 * Determine if the FBC watermark(s) can be used. IF
1377 * this isn't the case we prefer to disable the FBC
1378 * watermark(s) rather than disable the SR/HPLL
1379 * level(s) entirely. 'level-1' is the highest valid
1380 * level here.
1381 */
1382 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1383
1384 return 0;
1385}
1386
1387static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1388 struct intel_crtc *crtc)
1389{
1390 struct intel_crtc_state *crtc_state =
1391 intel_atomic_get_new_crtc_state(state, crtc);
1392 const struct intel_plane_state *old_plane_state;
1393 const struct intel_plane_state *new_plane_state;
1394 struct intel_plane *plane;
1395 unsigned int dirty = 0;
1396 int i;
1397
1398 for_each_oldnew_intel_plane_in_state(state, plane,
1399 old_plane_state,
1400 new_plane_state, i) {
1401 if (new_plane_state->hw.crtc != &crtc->base &&
1402 old_plane_state->hw.crtc != &crtc->base)
1403 continue;
1404
1405 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1406 dirty |= BIT(plane->id);
1407 }
1408
1409 if (!dirty)
1410 return 0;
1411
1412 return _g4x_compute_pipe_wm(crtc_state);
1413}
1414
1415static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1416 struct intel_crtc *crtc)
1417{
1418 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1419 struct intel_crtc_state *new_crtc_state =
1420 intel_atomic_get_new_crtc_state(state, crtc);
1421 const struct intel_crtc_state *old_crtc_state =
1422 intel_atomic_get_old_crtc_state(state, crtc);
1423 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1424 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1425 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1426 enum plane_id plane_id;
1427
1428 if (!new_crtc_state->hw.active ||
1429 intel_crtc_needs_modeset(new_crtc_state)) {
1430 *intermediate = *optimal;
1431
1432 intermediate->cxsr = false;
1433 intermediate->hpll_en = false;
1434 goto out;
1435 }
1436
1437 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1438 !new_crtc_state->disable_cxsr;
1439 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1440 !new_crtc_state->disable_cxsr;
1441 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1442
1443 for_each_plane_id_on_crtc(crtc, plane_id) {
1444 intermediate->wm.plane[plane_id] =
1445 max(optimal->wm.plane[plane_id],
1446 active->wm.plane[plane_id]);
1447
1448 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1449 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1450 }
1451
1452 intermediate->sr.plane = max(optimal->sr.plane,
1453 active->sr.plane);
1454 intermediate->sr.cursor = max(optimal->sr.cursor,
1455 active->sr.cursor);
1456 intermediate->sr.fbc = max(optimal->sr.fbc,
1457 active->sr.fbc);
1458
1459 intermediate->hpll.plane = max(optimal->hpll.plane,
1460 active->hpll.plane);
1461 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1462 active->hpll.cursor);
1463 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1464 active->hpll.fbc);
1465
1466 drm_WARN_ON(&dev_priv->drm,
1467 (intermediate->sr.plane >
1468 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1469 intermediate->sr.cursor >
1470 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1471 intermediate->cxsr);
1472 drm_WARN_ON(&dev_priv->drm,
1473 (intermediate->sr.plane >
1474 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1475 intermediate->sr.cursor >
1476 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1477 intermediate->hpll_en);
1478
1479 drm_WARN_ON(&dev_priv->drm,
1480 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1481 intermediate->fbc_en && intermediate->cxsr);
1482 drm_WARN_ON(&dev_priv->drm,
1483 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1484 intermediate->fbc_en && intermediate->hpll_en);
1485
1486out:
1487 /*
1488 * If our intermediate WM are identical to the final WM, then we can
1489 * omit the post-vblank programming; only update if it's different.
1490 */
1491 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1492 new_crtc_state->wm.need_postvbl_update = true;
1493
1494 return 0;
1495}
1496
1497static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1498 struct g4x_wm_values *wm)
1499{
1500 struct intel_crtc *crtc;
1501 int num_active_pipes = 0;
1502
1503 wm->cxsr = true;
1504 wm->hpll_en = true;
1505 wm->fbc_en = true;
1506
1507 for_each_intel_crtc(&dev_priv->drm, crtc) {
1508 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1509
1510 if (!crtc->active)
1511 continue;
1512
1513 if (!wm_state->cxsr)
1514 wm->cxsr = false;
1515 if (!wm_state->hpll_en)
1516 wm->hpll_en = false;
1517 if (!wm_state->fbc_en)
1518 wm->fbc_en = false;
1519
1520 num_active_pipes++;
1521 }
1522
1523 if (num_active_pipes != 1) {
1524 wm->cxsr = false;
1525 wm->hpll_en = false;
1526 wm->fbc_en = false;
1527 }
1528
1529 for_each_intel_crtc(&dev_priv->drm, crtc) {
1530 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1531 enum pipe pipe = crtc->pipe;
1532
1533 wm->pipe[pipe] = wm_state->wm;
1534 if (crtc->active && wm->cxsr)
1535 wm->sr = wm_state->sr;
1536 if (crtc->active && wm->hpll_en)
1537 wm->hpll = wm_state->hpll;
1538 }
1539}
1540
1541static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1542{
1543 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1544 struct g4x_wm_values new_wm = {};
1545
1546 g4x_merge_wm(dev_priv, &new_wm);
1547
1548 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1549 return;
1550
1551 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1552 _intel_set_memory_cxsr(dev_priv, false);
1553
1554 g4x_write_wm_values(dev_priv, &new_wm);
1555
1556 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1557 _intel_set_memory_cxsr(dev_priv, true);
1558
1559 *old_wm = new_wm;
1560}
1561
1562static void g4x_initial_watermarks(struct intel_atomic_state *state,
1563 struct intel_crtc *crtc)
1564{
1565 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1566 const struct intel_crtc_state *crtc_state =
1567 intel_atomic_get_new_crtc_state(state, crtc);
1568
1569 mutex_lock(&dev_priv->display.wm.wm_mutex);
1570 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1571 g4x_program_watermarks(dev_priv);
1572 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1573}
1574
1575static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1576 struct intel_crtc *crtc)
1577{
1578 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1579 const struct intel_crtc_state *crtc_state =
1580 intel_atomic_get_new_crtc_state(state, crtc);
1581
1582 if (!crtc_state->wm.need_postvbl_update)
1583 return;
1584
1585 mutex_lock(&dev_priv->display.wm.wm_mutex);
1586 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1587 g4x_program_watermarks(dev_priv);
1588 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1589}
1590
1591/* latency must be in 0.1us units. */
1592static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1593 unsigned int htotal,
1594 unsigned int width,
1595 unsigned int cpp,
1596 unsigned int latency)
1597{
1598 unsigned int ret;
1599
1600 ret = intel_wm_method2(pixel_rate, htotal,
1601 width, cpp, latency);
1602 ret = DIV_ROUND_UP(ret, 64);
1603
1604 return ret;
1605}
1606
1607static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1608{
1609 /* all latencies in usec */
1610 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1611
1612 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1613
1614 if (IS_CHERRYVIEW(dev_priv)) {
1615 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1616 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1617
1618 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1619 }
1620}
1621
1622static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1623 const struct intel_plane_state *plane_state,
1624 int level)
1625{
1626 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1627 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1628 const struct drm_display_mode *pipe_mode =
1629 &crtc_state->hw.pipe_mode;
1630 unsigned int pixel_rate, htotal, cpp, width, wm;
1631
1632 if (dev_priv->display.wm.pri_latency[level] == 0)
1633 return USHRT_MAX;
1634
1635 if (!intel_wm_plane_visible(crtc_state, plane_state))
1636 return 0;
1637
1638 cpp = plane_state->hw.fb->format->cpp[0];
1639 pixel_rate = crtc_state->pixel_rate;
1640 htotal = pipe_mode->crtc_htotal;
1641 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1642
1643 if (plane->id == PLANE_CURSOR) {
1644 /*
1645 * FIXME the formula gives values that are
1646 * too big for the cursor FIFO, and hence we
1647 * would never be able to use cursors. For
1648 * now just hardcode the watermark.
1649 */
1650 wm = 63;
1651 } else {
1652 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1653 dev_priv->display.wm.pri_latency[level] * 10);
1654 }
1655
1656 return min_t(unsigned int, wm, USHRT_MAX);
1657}
1658
1659static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1660{
1661 return (active_planes & (BIT(PLANE_SPRITE0) |
1662 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1663}
1664
1665static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1666{
1667 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1668 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1669 const struct g4x_pipe_wm *raw =
1670 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1671 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1672 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1673 int num_active_planes = hweight8(active_planes);
1674 const int fifo_size = 511;
1675 int fifo_extra, fifo_left = fifo_size;
1676 int sprite0_fifo_extra = 0;
1677 unsigned int total_rate;
1678 enum plane_id plane_id;
1679
1680 /*
1681 * When enabling sprite0 after sprite1 has already been enabled
1682 * we tend to get an underrun unless sprite0 already has some
1683 * FIFO space allcoated. Hence we always allocate at least one
1684 * cacheline for sprite0 whenever sprite1 is enabled.
1685 *
1686 * All other plane enable sequences appear immune to this problem.
1687 */
1688 if (vlv_need_sprite0_fifo_workaround(active_planes))
1689 sprite0_fifo_extra = 1;
1690
1691 total_rate = raw->plane[PLANE_PRIMARY] +
1692 raw->plane[PLANE_SPRITE0] +
1693 raw->plane[PLANE_SPRITE1] +
1694 sprite0_fifo_extra;
1695
1696 if (total_rate > fifo_size)
1697 return -EINVAL;
1698
1699 if (total_rate == 0)
1700 total_rate = 1;
1701
1702 for_each_plane_id_on_crtc(crtc, plane_id) {
1703 unsigned int rate;
1704
1705 if ((active_planes & BIT(plane_id)) == 0) {
1706 fifo_state->plane[plane_id] = 0;
1707 continue;
1708 }
1709
1710 rate = raw->plane[plane_id];
1711 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1712 fifo_left -= fifo_state->plane[plane_id];
1713 }
1714
1715 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1716 fifo_left -= sprite0_fifo_extra;
1717
1718 fifo_state->plane[PLANE_CURSOR] = 63;
1719
1720 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1721
1722 /* spread the remainder evenly */
1723 for_each_plane_id_on_crtc(crtc, plane_id) {
1724 int plane_extra;
1725
1726 if (fifo_left == 0)
1727 break;
1728
1729 if ((active_planes & BIT(plane_id)) == 0)
1730 continue;
1731
1732 plane_extra = min(fifo_extra, fifo_left);
1733 fifo_state->plane[plane_id] += plane_extra;
1734 fifo_left -= plane_extra;
1735 }
1736
1737 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1738
1739 /* give it all to the first plane if none are active */
1740 if (active_planes == 0) {
1741 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1742 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1743 }
1744
1745 return 0;
1746}
1747
1748/* mark all levels starting from 'level' as invalid */
1749static void vlv_invalidate_wms(struct intel_crtc *crtc,
1750 struct vlv_wm_state *wm_state, int level)
1751{
1752 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1753
1754 for (; level < intel_wm_num_levels(dev_priv); level++) {
1755 enum plane_id plane_id;
1756
1757 for_each_plane_id_on_crtc(crtc, plane_id)
1758 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1759
1760 wm_state->sr[level].cursor = USHRT_MAX;
1761 wm_state->sr[level].plane = USHRT_MAX;
1762 }
1763}
1764
1765static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1766{
1767 if (wm > fifo_size)
1768 return USHRT_MAX;
1769 else
1770 return fifo_size - wm;
1771}
1772
1773/*
1774 * Starting from 'level' set all higher
1775 * levels to 'value' in the "raw" watermarks.
1776 */
1777static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1778 int level, enum plane_id plane_id, u16 value)
1779{
1780 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1781 int num_levels = intel_wm_num_levels(dev_priv);
1782 bool dirty = false;
1783
1784 for (; level < num_levels; level++) {
1785 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1786
1787 dirty |= raw->plane[plane_id] != value;
1788 raw->plane[plane_id] = value;
1789 }
1790
1791 return dirty;
1792}
1793
1794static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1795 const struct intel_plane_state *plane_state)
1796{
1797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1798 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799 enum plane_id plane_id = plane->id;
1800 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801 int level;
1802 bool dirty = false;
1803
1804 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1805 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806 goto out;
1807 }
1808
1809 for (level = 0; level < num_levels; level++) {
1810 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1811 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1812 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1813
1814 if (wm > max_wm)
1815 break;
1816
1817 dirty |= raw->plane[plane_id] != wm;
1818 raw->plane[plane_id] = wm;
1819 }
1820
1821 /* mark all higher levels as invalid */
1822 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823
1824out:
1825 if (dirty)
1826 drm_dbg_kms(&dev_priv->drm,
1827 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1828 plane->base.name,
1829 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1830 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1831 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1832
1833 return dirty;
1834}
1835
1836static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1837 enum plane_id plane_id, int level)
1838{
1839 const struct g4x_pipe_wm *raw =
1840 &crtc_state->wm.vlv.raw[level];
1841 const struct vlv_fifo_state *fifo_state =
1842 &crtc_state->wm.vlv.fifo_state;
1843
1844 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1845}
1846
1847static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1848{
1849 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1850 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1851 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1852 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1853}
1854
1855static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1856{
1857 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1858 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1860 const struct vlv_fifo_state *fifo_state =
1861 &crtc_state->wm.vlv.fifo_state;
1862 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1863 int num_active_planes = hweight8(active_planes);
1864 enum plane_id plane_id;
1865 int level;
1866
1867 /* initially allow all levels */
1868 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1869 /*
1870 * Note that enabling cxsr with no primary/sprite planes
1871 * enabled can wedge the pipe. Hence we only allow cxsr
1872 * with exactly one enabled primary/sprite plane.
1873 */
1874 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1875
1876 for (level = 0; level < wm_state->num_levels; level++) {
1877 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1878 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1879
1880 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1881 break;
1882
1883 for_each_plane_id_on_crtc(crtc, plane_id) {
1884 wm_state->wm[level].plane[plane_id] =
1885 vlv_invert_wm_value(raw->plane[plane_id],
1886 fifo_state->plane[plane_id]);
1887 }
1888
1889 wm_state->sr[level].plane =
1890 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1891 raw->plane[PLANE_SPRITE0],
1892 raw->plane[PLANE_SPRITE1]),
1893 sr_fifo_size);
1894
1895 wm_state->sr[level].cursor =
1896 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1897 63);
1898 }
1899
1900 if (level == 0)
1901 return -EINVAL;
1902
1903 /* limit to only levels we can actually handle */
1904 wm_state->num_levels = level;
1905
1906 /* invalidate the higher levels */
1907 vlv_invalidate_wms(crtc, wm_state, level);
1908
1909 return 0;
1910}
1911
1912static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1913 struct intel_crtc *crtc)
1914{
1915 struct intel_crtc_state *crtc_state =
1916 intel_atomic_get_new_crtc_state(state, crtc);
1917 const struct intel_plane_state *old_plane_state;
1918 const struct intel_plane_state *new_plane_state;
1919 struct intel_plane *plane;
1920 unsigned int dirty = 0;
1921 int i;
1922
1923 for_each_oldnew_intel_plane_in_state(state, plane,
1924 old_plane_state,
1925 new_plane_state, i) {
1926 if (new_plane_state->hw.crtc != &crtc->base &&
1927 old_plane_state->hw.crtc != &crtc->base)
1928 continue;
1929
1930 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1931 dirty |= BIT(plane->id);
1932 }
1933
1934 /*
1935 * DSPARB registers may have been reset due to the
1936 * power well being turned off. Make sure we restore
1937 * them to a consistent state even if no primary/sprite
1938 * planes are initially active. We also force a FIFO
1939 * recomputation so that we are sure to sanitize the
1940 * FIFO setting we took over from the BIOS even if there
1941 * are no active planes on the crtc.
1942 */
1943 if (intel_crtc_needs_modeset(crtc_state))
1944 dirty = ~0;
1945
1946 if (!dirty)
1947 return 0;
1948
1949 /* cursor changes don't warrant a FIFO recompute */
1950 if (dirty & ~BIT(PLANE_CURSOR)) {
1951 const struct intel_crtc_state *old_crtc_state =
1952 intel_atomic_get_old_crtc_state(state, crtc);
1953 const struct vlv_fifo_state *old_fifo_state =
1954 &old_crtc_state->wm.vlv.fifo_state;
1955 const struct vlv_fifo_state *new_fifo_state =
1956 &crtc_state->wm.vlv.fifo_state;
1957 int ret;
1958
1959 ret = vlv_compute_fifo(crtc_state);
1960 if (ret)
1961 return ret;
1962
1963 if (intel_crtc_needs_modeset(crtc_state) ||
1964 memcmp(old_fifo_state, new_fifo_state,
1965 sizeof(*new_fifo_state)) != 0)
1966 crtc_state->fifo_changed = true;
1967 }
1968
1969 return _vlv_compute_pipe_wm(crtc_state);
1970}
1971
1972#define VLV_FIFO(plane, value) \
1973 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1974
1975static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1976 struct intel_crtc *crtc)
1977{
1978 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1979 struct intel_uncore *uncore = &dev_priv->uncore;
1980 const struct intel_crtc_state *crtc_state =
1981 intel_atomic_get_new_crtc_state(state, crtc);
1982 const struct vlv_fifo_state *fifo_state =
1983 &crtc_state->wm.vlv.fifo_state;
1984 int sprite0_start, sprite1_start, fifo_size;
1985 u32 dsparb, dsparb2, dsparb3;
1986
1987 if (!crtc_state->fifo_changed)
1988 return;
1989
1990 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1991 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1992 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1993
1994 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1995 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1996
1997 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1998
1999 /*
2000 * uncore.lock serves a double purpose here. It allows us to
2001 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2002 * it protects the DSPARB registers from getting clobbered by
2003 * parallel updates from multiple pipes.
2004 *
2005 * intel_pipe_update_start() has already disabled interrupts
2006 * for us, so a plain spin_lock() is sufficient here.
2007 */
2008 spin_lock(&uncore->lock);
2009
2010 switch (crtc->pipe) {
2011 case PIPE_A:
2012 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2013 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2014
2015 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2016 VLV_FIFO(SPRITEB, 0xff));
2017 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2018 VLV_FIFO(SPRITEB, sprite1_start));
2019
2020 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2021 VLV_FIFO(SPRITEB_HI, 0x1));
2022 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2023 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2024
2025 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2026 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2027 break;
2028 case PIPE_B:
2029 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2030 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2031
2032 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2033 VLV_FIFO(SPRITED, 0xff));
2034 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2035 VLV_FIFO(SPRITED, sprite1_start));
2036
2037 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2038 VLV_FIFO(SPRITED_HI, 0xff));
2039 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2040 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2041
2042 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2043 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2044 break;
2045 case PIPE_C:
2046 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2047 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2048
2049 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2050 VLV_FIFO(SPRITEF, 0xff));
2051 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2052 VLV_FIFO(SPRITEF, sprite1_start));
2053
2054 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2055 VLV_FIFO(SPRITEF_HI, 0xff));
2056 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2057 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2058
2059 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2060 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2061 break;
2062 default:
2063 break;
2064 }
2065
2066 intel_uncore_posting_read_fw(uncore, DSPARB);
2067
2068 spin_unlock(&uncore->lock);
2069}
2070
2071#undef VLV_FIFO
2072
2073static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2074 struct intel_crtc *crtc)
2075{
2076 struct intel_crtc_state *new_crtc_state =
2077 intel_atomic_get_new_crtc_state(state, crtc);
2078 const struct intel_crtc_state *old_crtc_state =
2079 intel_atomic_get_old_crtc_state(state, crtc);
2080 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2081 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2082 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2083 int level;
2084
2085 if (!new_crtc_state->hw.active ||
2086 intel_crtc_needs_modeset(new_crtc_state)) {
2087 *intermediate = *optimal;
2088
2089 intermediate->cxsr = false;
2090 goto out;
2091 }
2092
2093 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2094 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2095 !new_crtc_state->disable_cxsr;
2096
2097 for (level = 0; level < intermediate->num_levels; level++) {
2098 enum plane_id plane_id;
2099
2100 for_each_plane_id_on_crtc(crtc, plane_id) {
2101 intermediate->wm[level].plane[plane_id] =
2102 min(optimal->wm[level].plane[plane_id],
2103 active->wm[level].plane[plane_id]);
2104 }
2105
2106 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2107 active->sr[level].plane);
2108 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2109 active->sr[level].cursor);
2110 }
2111
2112 vlv_invalidate_wms(crtc, intermediate, level);
2113
2114out:
2115 /*
2116 * If our intermediate WM are identical to the final WM, then we can
2117 * omit the post-vblank programming; only update if it's different.
2118 */
2119 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2120 new_crtc_state->wm.need_postvbl_update = true;
2121
2122 return 0;
2123}
2124
2125static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2126 struct vlv_wm_values *wm)
2127{
2128 struct intel_crtc *crtc;
2129 int num_active_pipes = 0;
2130
2131 wm->level = dev_priv->display.wm.max_level;
2132 wm->cxsr = true;
2133
2134 for_each_intel_crtc(&dev_priv->drm, crtc) {
2135 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2136
2137 if (!crtc->active)
2138 continue;
2139
2140 if (!wm_state->cxsr)
2141 wm->cxsr = false;
2142
2143 num_active_pipes++;
2144 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2145 }
2146
2147 if (num_active_pipes != 1)
2148 wm->cxsr = false;
2149
2150 if (num_active_pipes > 1)
2151 wm->level = VLV_WM_LEVEL_PM2;
2152
2153 for_each_intel_crtc(&dev_priv->drm, crtc) {
2154 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2155 enum pipe pipe = crtc->pipe;
2156
2157 wm->pipe[pipe] = wm_state->wm[wm->level];
2158 if (crtc->active && wm->cxsr)
2159 wm->sr = wm_state->sr[wm->level];
2160
2161 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2162 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2163 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2164 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2165 }
2166}
2167
2168static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2169{
2170 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2171 struct vlv_wm_values new_wm = {};
2172
2173 vlv_merge_wm(dev_priv, &new_wm);
2174
2175 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2176 return;
2177
2178 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2179 chv_set_memory_dvfs(dev_priv, false);
2180
2181 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2182 chv_set_memory_pm5(dev_priv, false);
2183
2184 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2185 _intel_set_memory_cxsr(dev_priv, false);
2186
2187 vlv_write_wm_values(dev_priv, &new_wm);
2188
2189 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2190 _intel_set_memory_cxsr(dev_priv, true);
2191
2192 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2193 chv_set_memory_pm5(dev_priv, true);
2194
2195 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2196 chv_set_memory_dvfs(dev_priv, true);
2197
2198 *old_wm = new_wm;
2199}
2200
2201static void vlv_initial_watermarks(struct intel_atomic_state *state,
2202 struct intel_crtc *crtc)
2203{
2204 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2205 const struct intel_crtc_state *crtc_state =
2206 intel_atomic_get_new_crtc_state(state, crtc);
2207
2208 mutex_lock(&dev_priv->display.wm.wm_mutex);
2209 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2210 vlv_program_watermarks(dev_priv);
2211 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2212}
2213
2214static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2215 struct intel_crtc *crtc)
2216{
2217 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2218 const struct intel_crtc_state *crtc_state =
2219 intel_atomic_get_new_crtc_state(state, crtc);
2220
2221 if (!crtc_state->wm.need_postvbl_update)
2222 return;
2223
2224 mutex_lock(&dev_priv->display.wm.wm_mutex);
2225 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2226 vlv_program_watermarks(dev_priv);
2227 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2228}
2229
2230static void i965_update_wm(struct drm_i915_private *dev_priv)
2231{
2232 struct intel_crtc *crtc;
2233 int srwm = 1;
2234 int cursor_sr = 16;
2235 bool cxsr_enabled;
2236
2237 /* Calc sr entries for one plane configs */
2238 crtc = single_enabled_crtc(dev_priv);
2239 if (crtc) {
2240 /* self-refresh has much higher latency */
2241 static const int sr_latency_ns = 12000;
2242 const struct drm_display_mode *pipe_mode =
2243 &crtc->config->hw.pipe_mode;
2244 const struct drm_framebuffer *fb =
2245 crtc->base.primary->state->fb;
2246 int pixel_rate = crtc->config->pixel_rate;
2247 int htotal = pipe_mode->crtc_htotal;
2248 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2249 int cpp = fb->format->cpp[0];
2250 int entries;
2251
2252 entries = intel_wm_method2(pixel_rate, htotal,
2253 width, cpp, sr_latency_ns / 100);
2254 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2255 srwm = I965_FIFO_SIZE - entries;
2256 if (srwm < 0)
2257 srwm = 1;
2258 srwm &= 0x1ff;
2259 drm_dbg_kms(&dev_priv->drm,
2260 "self-refresh entries: %d, wm: %d\n",
2261 entries, srwm);
2262
2263 entries = intel_wm_method2(pixel_rate, htotal,
2264 crtc->base.cursor->state->crtc_w, 4,
2265 sr_latency_ns / 100);
2266 entries = DIV_ROUND_UP(entries,
2267 i965_cursor_wm_info.cacheline_size) +
2268 i965_cursor_wm_info.guard_size;
2269
2270 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2271 if (cursor_sr > i965_cursor_wm_info.max_wm)
2272 cursor_sr = i965_cursor_wm_info.max_wm;
2273
2274 drm_dbg_kms(&dev_priv->drm,
2275 "self-refresh watermark: display plane %d "
2276 "cursor %d\n", srwm, cursor_sr);
2277
2278 cxsr_enabled = true;
2279 } else {
2280 cxsr_enabled = false;
2281 /* Turn off self refresh if both pipes are enabled */
2282 intel_set_memory_cxsr(dev_priv, false);
2283 }
2284
2285 drm_dbg_kms(&dev_priv->drm,
2286 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2287 srwm);
2288
2289 /* 965 has limitations... */
2290 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2291 FW_WM(8, CURSORB) |
2292 FW_WM(8, PLANEB) |
2293 FW_WM(8, PLANEA));
2294 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2295 FW_WM(8, PLANEC_OLD));
2296 /* update cursor SR watermark */
2297 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2298
2299 if (cxsr_enabled)
2300 intel_set_memory_cxsr(dev_priv, true);
2301}
2302
2303#undef FW_WM
2304
2305static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2306 enum i9xx_plane_id i9xx_plane)
2307{
2308 struct intel_plane *plane;
2309
2310 for_each_intel_plane(&i915->drm, plane) {
2311 if (plane->id == PLANE_PRIMARY &&
2312 plane->i9xx_plane == i9xx_plane)
2313 return intel_crtc_for_pipe(i915, plane->pipe);
2314 }
2315
2316 return NULL;
2317}
2318
2319static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2320{
2321 const struct intel_watermark_params *wm_info;
2322 u32 fwater_lo;
2323 u32 fwater_hi;
2324 int cwm, srwm = 1;
2325 int fifo_size;
2326 int planea_wm, planeb_wm;
2327 struct intel_crtc *crtc;
2328
2329 if (IS_I945GM(dev_priv))
2330 wm_info = &i945_wm_info;
2331 else if (DISPLAY_VER(dev_priv) != 2)
2332 wm_info = &i915_wm_info;
2333 else
2334 wm_info = &i830_a_wm_info;
2335
2336 if (DISPLAY_VER(dev_priv) == 2)
2337 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2338 else
2339 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2340 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2341 if (intel_crtc_active(crtc)) {
2342 const struct drm_framebuffer *fb =
2343 crtc->base.primary->state->fb;
2344 int cpp;
2345
2346 if (DISPLAY_VER(dev_priv) == 2)
2347 cpp = 4;
2348 else
2349 cpp = fb->format->cpp[0];
2350
2351 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2352 wm_info, fifo_size, cpp,
2353 pessimal_latency_ns);
2354 } else {
2355 planea_wm = fifo_size - wm_info->guard_size;
2356 if (planea_wm > (long)wm_info->max_wm)
2357 planea_wm = wm_info->max_wm;
2358 }
2359
2360 if (DISPLAY_VER(dev_priv) == 2)
2361 wm_info = &i830_bc_wm_info;
2362
2363 if (DISPLAY_VER(dev_priv) == 2)
2364 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2365 else
2366 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2367 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2368 if (intel_crtc_active(crtc)) {
2369 const struct drm_framebuffer *fb =
2370 crtc->base.primary->state->fb;
2371 int cpp;
2372
2373 if (DISPLAY_VER(dev_priv) == 2)
2374 cpp = 4;
2375 else
2376 cpp = fb->format->cpp[0];
2377
2378 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379 wm_info, fifo_size, cpp,
2380 pessimal_latency_ns);
2381 } else {
2382 planeb_wm = fifo_size - wm_info->guard_size;
2383 if (planeb_wm > (long)wm_info->max_wm)
2384 planeb_wm = wm_info->max_wm;
2385 }
2386
2387 drm_dbg_kms(&dev_priv->drm,
2388 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2389
2390 crtc = single_enabled_crtc(dev_priv);
2391 if (IS_I915GM(dev_priv) && crtc) {
2392 struct drm_i915_gem_object *obj;
2393
2394 obj = intel_fb_obj(crtc->base.primary->state->fb);
2395
2396 /* self-refresh seems busted with untiled */
2397 if (!i915_gem_object_is_tiled(obj))
2398 crtc = NULL;
2399 }
2400
2401 /*
2402 * Overlay gets an aggressive default since video jitter is bad.
2403 */
2404 cwm = 2;
2405
2406 /* Play safe and disable self-refresh before adjusting watermarks. */
2407 intel_set_memory_cxsr(dev_priv, false);
2408
2409 /* Calc sr entries for one plane configs */
2410 if (HAS_FW_BLC(dev_priv) && crtc) {
2411 /* self-refresh has much higher latency */
2412 static const int sr_latency_ns = 6000;
2413 const struct drm_display_mode *pipe_mode =
2414 &crtc->config->hw.pipe_mode;
2415 const struct drm_framebuffer *fb =
2416 crtc->base.primary->state->fb;
2417 int pixel_rate = crtc->config->pixel_rate;
2418 int htotal = pipe_mode->crtc_htotal;
2419 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2420 int cpp;
2421 int entries;
2422
2423 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2424 cpp = 4;
2425 else
2426 cpp = fb->format->cpp[0];
2427
2428 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2429 sr_latency_ns / 100);
2430 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2431 drm_dbg_kms(&dev_priv->drm,
2432 "self-refresh entries: %d\n", entries);
2433 srwm = wm_info->fifo_size - entries;
2434 if (srwm < 0)
2435 srwm = 1;
2436
2437 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2438 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2439 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2440 else
2441 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2442 }
2443
2444 drm_dbg_kms(&dev_priv->drm,
2445 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2446 planea_wm, planeb_wm, cwm, srwm);
2447
2448 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2449 fwater_hi = (cwm & 0x1f);
2450
2451 /* Set request length to 8 cachelines per fetch */
2452 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2453 fwater_hi = fwater_hi | (1 << 8);
2454
2455 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2456 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2457
2458 if (crtc)
2459 intel_set_memory_cxsr(dev_priv, true);
2460}
2461
2462static void i845_update_wm(struct drm_i915_private *dev_priv)
2463{
2464 struct intel_crtc *crtc;
2465 u32 fwater_lo;
2466 int planea_wm;
2467
2468 crtc = single_enabled_crtc(dev_priv);
2469 if (crtc == NULL)
2470 return;
2471
2472 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2473 &i845_wm_info,
2474 i845_get_fifo_size(dev_priv, PLANE_A),
2475 4, pessimal_latency_ns);
2476 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2477 fwater_lo |= (3<<8) | planea_wm;
2478
2479 drm_dbg_kms(&dev_priv->drm,
2480 "Setting FIFO watermarks - A: %d\n", planea_wm);
2481
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483}
2484
2485/* latency must be in 0.1us units. */
2486static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2487 unsigned int cpp,
2488 unsigned int latency)
2489{
2490 unsigned int ret;
2491
2492 ret = intel_wm_method1(pixel_rate, cpp, latency);
2493 ret = DIV_ROUND_UP(ret, 64) + 2;
2494
2495 return ret;
2496}
2497
2498/* latency must be in 0.1us units. */
2499static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2500 unsigned int htotal,
2501 unsigned int width,
2502 unsigned int cpp,
2503 unsigned int latency)
2504{
2505 unsigned int ret;
2506
2507 ret = intel_wm_method2(pixel_rate, htotal,
2508 width, cpp, latency);
2509 ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511 return ret;
2512}
2513
2514static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2515{
2516 /*
2517 * Neither of these should be possible since this function shouldn't be
2518 * called if the CRTC is off or the plane is invisible. But let's be
2519 * extra paranoid to avoid a potential divide-by-zero if we screw up
2520 * elsewhere in the driver.
2521 */
2522 if (WARN_ON(!cpp))
2523 return 0;
2524 if (WARN_ON(!horiz_pixels))
2525 return 0;
2526
2527 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2528}
2529
2530struct ilk_wm_maximums {
2531 u16 pri;
2532 u16 spr;
2533 u16 cur;
2534 u16 fbc;
2535};
2536
2537/*
2538 * For both WM_PIPE and WM_LP.
2539 * mem_value must be in 0.1us units.
2540 */
2541static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2542 const struct intel_plane_state *plane_state,
2543 u32 mem_value, bool is_lp)
2544{
2545 u32 method1, method2;
2546 int cpp;
2547
2548 if (mem_value == 0)
2549 return U32_MAX;
2550
2551 if (!intel_wm_plane_visible(crtc_state, plane_state))
2552 return 0;
2553
2554 cpp = plane_state->hw.fb->format->cpp[0];
2555
2556 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2557
2558 if (!is_lp)
2559 return method1;
2560
2561 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2562 crtc_state->hw.pipe_mode.crtc_htotal,
2563 drm_rect_width(&plane_state->uapi.src) >> 16,
2564 cpp, mem_value);
2565
2566 return min(method1, method2);
2567}
2568
2569/*
2570 * For both WM_PIPE and WM_LP.
2571 * mem_value must be in 0.1us units.
2572 */
2573static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2574 const struct intel_plane_state *plane_state,
2575 u32 mem_value)
2576{
2577 u32 method1, method2;
2578 int cpp;
2579
2580 if (mem_value == 0)
2581 return U32_MAX;
2582
2583 if (!intel_wm_plane_visible(crtc_state, plane_state))
2584 return 0;
2585
2586 cpp = plane_state->hw.fb->format->cpp[0];
2587
2588 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2589 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2590 crtc_state->hw.pipe_mode.crtc_htotal,
2591 drm_rect_width(&plane_state->uapi.src) >> 16,
2592 cpp, mem_value);
2593 return min(method1, method2);
2594}
2595
2596/*
2597 * For both WM_PIPE and WM_LP.
2598 * mem_value must be in 0.1us units.
2599 */
2600static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2601 const struct intel_plane_state *plane_state,
2602 u32 mem_value)
2603{
2604 int cpp;
2605
2606 if (mem_value == 0)
2607 return U32_MAX;
2608
2609 if (!intel_wm_plane_visible(crtc_state, plane_state))
2610 return 0;
2611
2612 cpp = plane_state->hw.fb->format->cpp[0];
2613
2614 return ilk_wm_method2(crtc_state->pixel_rate,
2615 crtc_state->hw.pipe_mode.crtc_htotal,
2616 drm_rect_width(&plane_state->uapi.src) >> 16,
2617 cpp, mem_value);
2618}
2619
2620/* Only for WM_LP. */
2621static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2622 const struct intel_plane_state *plane_state,
2623 u32 pri_val)
2624{
2625 int cpp;
2626
2627 if (!intel_wm_plane_visible(crtc_state, plane_state))
2628 return 0;
2629
2630 cpp = plane_state->hw.fb->format->cpp[0];
2631
2632 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2633 cpp);
2634}
2635
2636static unsigned int
2637ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2638{
2639 if (DISPLAY_VER(dev_priv) >= 8)
2640 return 3072;
2641 else if (DISPLAY_VER(dev_priv) >= 7)
2642 return 768;
2643 else
2644 return 512;
2645}
2646
2647static unsigned int
2648ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2649 int level, bool is_sprite)
2650{
2651 if (DISPLAY_VER(dev_priv) >= 8)
2652 /* BDW primary/sprite plane watermarks */
2653 return level == 0 ? 255 : 2047;
2654 else if (DISPLAY_VER(dev_priv) >= 7)
2655 /* IVB/HSW primary/sprite plane watermarks */
2656 return level == 0 ? 127 : 1023;
2657 else if (!is_sprite)
2658 /* ILK/SNB primary plane watermarks */
2659 return level == 0 ? 127 : 511;
2660 else
2661 /* ILK/SNB sprite plane watermarks */
2662 return level == 0 ? 63 : 255;
2663}
2664
2665static unsigned int
2666ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2667{
2668 if (DISPLAY_VER(dev_priv) >= 7)
2669 return level == 0 ? 63 : 255;
2670 else
2671 return level == 0 ? 31 : 63;
2672}
2673
2674static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2675{
2676 if (DISPLAY_VER(dev_priv) >= 8)
2677 return 31;
2678 else
2679 return 15;
2680}
2681
2682/* Calculate the maximum primary/sprite plane watermark */
2683static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2684 int level,
2685 const struct intel_wm_config *config,
2686 enum intel_ddb_partitioning ddb_partitioning,
2687 bool is_sprite)
2688{
2689 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2690
2691 /* if sprites aren't enabled, sprites get nothing */
2692 if (is_sprite && !config->sprites_enabled)
2693 return 0;
2694
2695 /* HSW allows LP1+ watermarks even with multiple pipes */
2696 if (level == 0 || config->num_pipes_active > 1) {
2697 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2698
2699 /*
2700 * For some reason the non self refresh
2701 * FIFO size is only half of the self
2702 * refresh FIFO size on ILK/SNB.
2703 */
2704 if (DISPLAY_VER(dev_priv) <= 6)
2705 fifo_size /= 2;
2706 }
2707
2708 if (config->sprites_enabled) {
2709 /* level 0 is always calculated with 1:1 split */
2710 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2711 if (is_sprite)
2712 fifo_size *= 5;
2713 fifo_size /= 6;
2714 } else {
2715 fifo_size /= 2;
2716 }
2717 }
2718
2719 /* clamp to max that the registers can hold */
2720 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2721}
2722
2723/* Calculate the maximum cursor plane watermark */
2724static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2725 int level,
2726 const struct intel_wm_config *config)
2727{
2728 /* HSW LP1+ watermarks w/ multiple pipes */
2729 if (level > 0 && config->num_pipes_active > 1)
2730 return 64;
2731
2732 /* otherwise just report max that registers can hold */
2733 return ilk_cursor_wm_reg_max(dev_priv, level);
2734}
2735
2736static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2737 int level,
2738 const struct intel_wm_config *config,
2739 enum intel_ddb_partitioning ddb_partitioning,
2740 struct ilk_wm_maximums *max)
2741{
2742 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2743 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2744 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2745 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2746}
2747
2748static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2749 int level,
2750 struct ilk_wm_maximums *max)
2751{
2752 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2753 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2754 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2755 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2756}
2757
2758static bool ilk_validate_wm_level(int level,
2759 const struct ilk_wm_maximums *max,
2760 struct intel_wm_level *result)
2761{
2762 bool ret;
2763
2764 /* already determined to be invalid? */
2765 if (!result->enable)
2766 return false;
2767
2768 result->enable = result->pri_val <= max->pri &&
2769 result->spr_val <= max->spr &&
2770 result->cur_val <= max->cur;
2771
2772 ret = result->enable;
2773
2774 /*
2775 * HACK until we can pre-compute everything,
2776 * and thus fail gracefully if LP0 watermarks
2777 * are exceeded...
2778 */
2779 if (level == 0 && !result->enable) {
2780 if (result->pri_val > max->pri)
2781 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2782 level, result->pri_val, max->pri);
2783 if (result->spr_val > max->spr)
2784 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2785 level, result->spr_val, max->spr);
2786 if (result->cur_val > max->cur)
2787 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2788 level, result->cur_val, max->cur);
2789
2790 result->pri_val = min_t(u32, result->pri_val, max->pri);
2791 result->spr_val = min_t(u32, result->spr_val, max->spr);
2792 result->cur_val = min_t(u32, result->cur_val, max->cur);
2793 result->enable = true;
2794 }
2795
2796 return ret;
2797}
2798
2799static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2800 const struct intel_crtc *crtc,
2801 int level,
2802 struct intel_crtc_state *crtc_state,
2803 const struct intel_plane_state *pristate,
2804 const struct intel_plane_state *sprstate,
2805 const struct intel_plane_state *curstate,
2806 struct intel_wm_level *result)
2807{
2808 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2809 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2810 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2811
2812 /* WM1+ latency values stored in 0.5us units */
2813 if (level > 0) {
2814 pri_latency *= 5;
2815 spr_latency *= 5;
2816 cur_latency *= 5;
2817 }
2818
2819 if (pristate) {
2820 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2821 pri_latency, level);
2822 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2823 }
2824
2825 if (sprstate)
2826 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2827
2828 if (curstate)
2829 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2830
2831 result->enable = true;
2832}
2833
2834static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2835{
2836 u64 sskpd;
2837
2838 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2839
2840 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2841 if (wm[0] == 0)
2842 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2843 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2844 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2845 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2846 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2847}
2848
2849static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2850{
2851 u32 sskpd;
2852
2853 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2854
2855 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2856 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2857 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2858 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2859}
2860
2861static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2862{
2863 u32 mltr;
2864
2865 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2866
2867 /* ILK primary LP0 latency is 700 ns */
2868 wm[0] = 7;
2869 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2870 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2871}
2872
2873static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2874 u16 wm[5])
2875{
2876 /* ILK sprite LP0 latency is 1300 ns */
2877 if (DISPLAY_VER(dev_priv) == 5)
2878 wm[0] = 13;
2879}
2880
2881static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2882 u16 wm[5])
2883{
2884 /* ILK cursor LP0 latency is 1300 ns */
2885 if (DISPLAY_VER(dev_priv) == 5)
2886 wm[0] = 13;
2887}
2888
2889int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2890{
2891 /* how many WM levels are we expecting */
2892 if (HAS_HW_SAGV_WM(dev_priv))
2893 return 5;
2894 else if (DISPLAY_VER(dev_priv) >= 9)
2895 return 7;
2896 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2897 return 4;
2898 else if (DISPLAY_VER(dev_priv) >= 6)
2899 return 3;
2900 else
2901 return 2;
2902}
2903
2904void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2905 const char *name, const u16 wm[])
2906{
2907 int level, max_level = ilk_wm_max_level(dev_priv);
2908
2909 for (level = 0; level <= max_level; level++) {
2910 unsigned int latency = wm[level];
2911
2912 if (latency == 0) {
2913 drm_dbg_kms(&dev_priv->drm,
2914 "%s WM%d latency not provided\n",
2915 name, level);
2916 continue;
2917 }
2918
2919 /*
2920 * - latencies are in us on gen9.
2921 * - before then, WM1+ latency values are in 0.5us units
2922 */
2923 if (DISPLAY_VER(dev_priv) >= 9)
2924 latency *= 10;
2925 else if (level > 0)
2926 latency *= 5;
2927
2928 drm_dbg_kms(&dev_priv->drm,
2929 "%s WM%d latency %u (%u.%u usec)\n", name, level,
2930 wm[level], latency / 10, latency % 10);
2931 }
2932}
2933
2934static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2935 u16 wm[5], u16 min)
2936{
2937 int level, max_level = ilk_wm_max_level(dev_priv);
2938
2939 if (wm[0] >= min)
2940 return false;
2941
2942 wm[0] = max(wm[0], min);
2943 for (level = 1; level <= max_level; level++)
2944 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2945
2946 return true;
2947}
2948
2949static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2950{
2951 bool changed;
2952
2953 /*
2954 * The BIOS provided WM memory latency values are often
2955 * inadequate for high resolution displays. Adjust them.
2956 */
2957 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2958 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2959 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2960
2961 if (!changed)
2962 return;
2963
2964 drm_dbg_kms(&dev_priv->drm,
2965 "WM latency values increased to avoid potential underruns\n");
2966 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2967 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2968 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2969}
2970
2971static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2972{
2973 /*
2974 * On some SNB machines (Thinkpad X220 Tablet at least)
2975 * LP3 usage can cause vblank interrupts to be lost.
2976 * The DEIIR bit will go high but it looks like the CPU
2977 * never gets interrupted.
2978 *
2979 * It's not clear whether other interrupt source could
2980 * be affected or if this is somehow limited to vblank
2981 * interrupts only. To play it safe we disable LP3
2982 * watermarks entirely.
2983 */
2984 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2985 dev_priv->display.wm.spr_latency[3] == 0 &&
2986 dev_priv->display.wm.cur_latency[3] == 0)
2987 return;
2988
2989 dev_priv->display.wm.pri_latency[3] = 0;
2990 dev_priv->display.wm.spr_latency[3] = 0;
2991 dev_priv->display.wm.cur_latency[3] = 0;
2992
2993 drm_dbg_kms(&dev_priv->drm,
2994 "LP3 watermarks disabled due to potential for lost interrupts\n");
2995 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2996 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2997 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2998}
2999
3000static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3001{
3002 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3003 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3005 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3006 else
3007 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3008
3009 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3010 sizeof(dev_priv->display.wm.pri_latency));
3011 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3012 sizeof(dev_priv->display.wm.pri_latency));
3013
3014 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3015 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3016
3017 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3018 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3019 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3020
3021 if (DISPLAY_VER(dev_priv) == 6) {
3022 snb_wm_latency_quirk(dev_priv);
3023 snb_wm_lp3_irq_quirk(dev_priv);
3024 }
3025}
3026
3027static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3028 struct intel_pipe_wm *pipe_wm)
3029{
3030 /* LP0 watermark maximums depend on this pipe alone */
3031 const struct intel_wm_config config = {
3032 .num_pipes_active = 1,
3033 .sprites_enabled = pipe_wm->sprites_enabled,
3034 .sprites_scaled = pipe_wm->sprites_scaled,
3035 };
3036 struct ilk_wm_maximums max;
3037
3038 /* LP0 watermarks always use 1/2 DDB partitioning */
3039 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3040
3041 /* At least LP0 must be valid */
3042 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3043 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3044 return false;
3045 }
3046
3047 return true;
3048}
3049
3050/* Compute new watermarks for the pipe */
3051static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3052 struct intel_crtc *crtc)
3053{
3054 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3055 struct intel_crtc_state *crtc_state =
3056 intel_atomic_get_new_crtc_state(state, crtc);
3057 struct intel_pipe_wm *pipe_wm;
3058 struct intel_plane *plane;
3059 const struct intel_plane_state *plane_state;
3060 const struct intel_plane_state *pristate = NULL;
3061 const struct intel_plane_state *sprstate = NULL;
3062 const struct intel_plane_state *curstate = NULL;
3063 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3064 struct ilk_wm_maximums max;
3065
3066 pipe_wm = &crtc_state->wm.ilk.optimal;
3067
3068 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3069 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3070 pristate = plane_state;
3071 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3072 sprstate = plane_state;
3073 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3074 curstate = plane_state;
3075 }
3076
3077 pipe_wm->pipe_enabled = crtc_state->hw.active;
3078 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3079 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3080
3081 usable_level = max_level;
3082
3083 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3084 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3085 usable_level = 1;
3086
3087 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3088 if (pipe_wm->sprites_scaled)
3089 usable_level = 0;
3090
3091 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3092 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3093 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3094
3095 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3096 return -EINVAL;
3097
3098 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3099
3100 for (level = 1; level <= usable_level; level++) {
3101 struct intel_wm_level *wm = &pipe_wm->wm[level];
3102
3103 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3104 pristate, sprstate, curstate, wm);
3105
3106 /*
3107 * Disable any watermark level that exceeds the
3108 * register maximums since such watermarks are
3109 * always invalid.
3110 */
3111 if (!ilk_validate_wm_level(level, &max, wm)) {
3112 memset(wm, 0, sizeof(*wm));
3113 break;
3114 }
3115 }
3116
3117 return 0;
3118}
3119
3120/*
3121 * Build a set of 'intermediate' watermark values that satisfy both the old
3122 * state and the new state. These can be programmed to the hardware
3123 * immediately.
3124 */
3125static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3126 struct intel_crtc *crtc)
3127{
3128 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3129 struct intel_crtc_state *new_crtc_state =
3130 intel_atomic_get_new_crtc_state(state, crtc);
3131 const struct intel_crtc_state *old_crtc_state =
3132 intel_atomic_get_old_crtc_state(state, crtc);
3133 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3134 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3135 int level, max_level = ilk_wm_max_level(dev_priv);
3136
3137 /*
3138 * Start with the final, target watermarks, then combine with the
3139 * currently active watermarks to get values that are safe both before
3140 * and after the vblank.
3141 */
3142 *a = new_crtc_state->wm.ilk.optimal;
3143 if (!new_crtc_state->hw.active ||
3144 intel_crtc_needs_modeset(new_crtc_state) ||
3145 state->skip_intermediate_wm)
3146 return 0;
3147
3148 a->pipe_enabled |= b->pipe_enabled;
3149 a->sprites_enabled |= b->sprites_enabled;
3150 a->sprites_scaled |= b->sprites_scaled;
3151
3152 for (level = 0; level <= max_level; level++) {
3153 struct intel_wm_level *a_wm = &a->wm[level];
3154 const struct intel_wm_level *b_wm = &b->wm[level];
3155
3156 a_wm->enable &= b_wm->enable;
3157 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3158 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3159 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3160 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3161 }
3162
3163 /*
3164 * We need to make sure that these merged watermark values are
3165 * actually a valid configuration themselves. If they're not,
3166 * there's no safe way to transition from the old state to
3167 * the new state, so we need to fail the atomic transaction.
3168 */
3169 if (!ilk_validate_pipe_wm(dev_priv, a))
3170 return -EINVAL;
3171
3172 /*
3173 * If our intermediate WM are identical to the final WM, then we can
3174 * omit the post-vblank programming; only update if it's different.
3175 */
3176 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3177 new_crtc_state->wm.need_postvbl_update = true;
3178
3179 return 0;
3180}
3181
3182/*
3183 * Merge the watermarks from all active pipes for a specific level.
3184 */
3185static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3186 int level,
3187 struct intel_wm_level *ret_wm)
3188{
3189 const struct intel_crtc *crtc;
3190
3191 ret_wm->enable = true;
3192
3193 for_each_intel_crtc(&dev_priv->drm, crtc) {
3194 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3195 const struct intel_wm_level *wm = &active->wm[level];
3196
3197 if (!active->pipe_enabled)
3198 continue;
3199
3200 /*
3201 * The watermark values may have been used in the past,
3202 * so we must maintain them in the registers for some
3203 * time even if the level is now disabled.
3204 */
3205 if (!wm->enable)
3206 ret_wm->enable = false;
3207
3208 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3209 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3210 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3211 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3212 }
3213}
3214
3215/*
3216 * Merge all low power watermarks for all active pipes.
3217 */
3218static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3219 const struct intel_wm_config *config,
3220 const struct ilk_wm_maximums *max,
3221 struct intel_pipe_wm *merged)
3222{
3223 int level, max_level = ilk_wm_max_level(dev_priv);
3224 int last_enabled_level = max_level;
3225
3226 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3227 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3228 config->num_pipes_active > 1)
3229 last_enabled_level = 0;
3230
3231 /* ILK: FBC WM must be disabled always */
3232 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3233
3234 /* merge each WM1+ level */
3235 for (level = 1; level <= max_level; level++) {
3236 struct intel_wm_level *wm = &merged->wm[level];
3237
3238 ilk_merge_wm_level(dev_priv, level, wm);
3239
3240 if (level > last_enabled_level)
3241 wm->enable = false;
3242 else if (!ilk_validate_wm_level(level, max, wm))
3243 /* make sure all following levels get disabled */
3244 last_enabled_level = level - 1;
3245
3246 /*
3247 * The spec says it is preferred to disable
3248 * FBC WMs instead of disabling a WM level.
3249 */
3250 if (wm->fbc_val > max->fbc) {
3251 if (wm->enable)
3252 merged->fbc_wm_enabled = false;
3253 wm->fbc_val = 0;
3254 }
3255 }
3256
3257 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3258 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3259 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3260 for (level = 2; level <= max_level; level++) {
3261 struct intel_wm_level *wm = &merged->wm[level];
3262
3263 wm->enable = false;
3264 }
3265 }
3266}
3267
3268static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3269{
3270 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3271 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3272}
3273
3274/* The value we need to program into the WM_LPx latency field */
3275static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3276 int level)
3277{
3278 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3279 return 2 * level;
3280 else
3281 return dev_priv->display.wm.pri_latency[level];
3282}
3283
3284static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3285 const struct intel_pipe_wm *merged,
3286 enum intel_ddb_partitioning partitioning,
3287 struct ilk_wm_values *results)
3288{
3289 struct intel_crtc *crtc;
3290 int level, wm_lp;
3291
3292 results->enable_fbc_wm = merged->fbc_wm_enabled;
3293 results->partitioning = partitioning;
3294
3295 /* LP1+ register values */
3296 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3297 const struct intel_wm_level *r;
3298
3299 level = ilk_wm_lp_to_level(wm_lp, merged);
3300
3301 r = &merged->wm[level];
3302
3303 /*
3304 * Maintain the watermark values even if the level is
3305 * disabled. Doing otherwise could cause underruns.
3306 */
3307 results->wm_lp[wm_lp - 1] =
3308 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3309 WM_LP_PRIMARY(r->pri_val) |
3310 WM_LP_CURSOR(r->cur_val);
3311
3312 if (r->enable)
3313 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3314
3315 if (DISPLAY_VER(dev_priv) >= 8)
3316 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3317 else
3318 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3319
3320 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3321
3322 /*
3323 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3324 * level is disabled. Doing otherwise could cause underruns.
3325 */
3326 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3327 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3328 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3329 }
3330 }
3331
3332 /* LP0 register values */
3333 for_each_intel_crtc(&dev_priv->drm, crtc) {
3334 enum pipe pipe = crtc->pipe;
3335 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3336 const struct intel_wm_level *r = &pipe_wm->wm[0];
3337
3338 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3339 continue;
3340
3341 results->wm_pipe[pipe] =
3342 WM0_PIPE_PRIMARY(r->pri_val) |
3343 WM0_PIPE_SPRITE(r->spr_val) |
3344 WM0_PIPE_CURSOR(r->cur_val);
3345 }
3346}
3347
3348/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3349 * case both are at the same level. Prefer r1 in case they're the same. */
3350static struct intel_pipe_wm *
3351ilk_find_best_result(struct drm_i915_private *dev_priv,
3352 struct intel_pipe_wm *r1,
3353 struct intel_pipe_wm *r2)
3354{
3355 int level, max_level = ilk_wm_max_level(dev_priv);
3356 int level1 = 0, level2 = 0;
3357
3358 for (level = 1; level <= max_level; level++) {
3359 if (r1->wm[level].enable)
3360 level1 = level;
3361 if (r2->wm[level].enable)
3362 level2 = level;
3363 }
3364
3365 if (level1 == level2) {
3366 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3367 return r2;
3368 else
3369 return r1;
3370 } else if (level1 > level2) {
3371 return r1;
3372 } else {
3373 return r2;
3374 }
3375}
3376
3377/* dirty bits used to track which watermarks need changes */
3378#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3379#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3380#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3381#define WM_DIRTY_FBC (1 << 24)
3382#define WM_DIRTY_DDB (1 << 25)
3383
3384static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3385 const struct ilk_wm_values *old,
3386 const struct ilk_wm_values *new)
3387{
3388 unsigned int dirty = 0;
3389 enum pipe pipe;
3390 int wm_lp;
3391
3392 for_each_pipe(dev_priv, pipe) {
3393 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3394 dirty |= WM_DIRTY_PIPE(pipe);
3395 /* Must disable LP1+ watermarks too */
3396 dirty |= WM_DIRTY_LP_ALL;
3397 }
3398 }
3399
3400 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3401 dirty |= WM_DIRTY_FBC;
3402 /* Must disable LP1+ watermarks too */
3403 dirty |= WM_DIRTY_LP_ALL;
3404 }
3405
3406 if (old->partitioning != new->partitioning) {
3407 dirty |= WM_DIRTY_DDB;
3408 /* Must disable LP1+ watermarks too */
3409 dirty |= WM_DIRTY_LP_ALL;
3410 }
3411
3412 /* LP1+ watermarks already deemed dirty, no need to continue */
3413 if (dirty & WM_DIRTY_LP_ALL)
3414 return dirty;
3415
3416 /* Find the lowest numbered LP1+ watermark in need of an update... */
3417 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3418 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3419 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3420 break;
3421 }
3422
3423 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3424 for (; wm_lp <= 3; wm_lp++)
3425 dirty |= WM_DIRTY_LP(wm_lp);
3426
3427 return dirty;
3428}
3429
3430static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3431 unsigned int dirty)
3432{
3433 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3434 bool changed = false;
3435
3436 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3437 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3438 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3439 changed = true;
3440 }
3441 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3442 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3443 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3444 changed = true;
3445 }
3446 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3447 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3448 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3449 changed = true;
3450 }
3451
3452 /*
3453 * Don't touch WM_LP_SPRITE_ENABLE here.
3454 * Doing so could cause underruns.
3455 */
3456
3457 return changed;
3458}
3459
3460/*
3461 * The spec says we shouldn't write when we don't need, because every write
3462 * causes WMs to be re-evaluated, expending some power.
3463 */
3464static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3465 struct ilk_wm_values *results)
3466{
3467 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3468 unsigned int dirty;
3469
3470 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3471 if (!dirty)
3472 return;
3473
3474 _ilk_disable_lp_wm(dev_priv, dirty);
3475
3476 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3477 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3478 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3479 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3480 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3481 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3482
3483 if (dirty & WM_DIRTY_DDB) {
3484 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3485 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3486 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3487 WM_MISC_DATA_PARTITION_5_6);
3488 else
3489 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3490 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3491 DISP_DATA_PARTITION_5_6);
3492 }
3493
3494 if (dirty & WM_DIRTY_FBC)
3495 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3496 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3497
3498 if (dirty & WM_DIRTY_LP(1) &&
3499 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3500 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3501
3502 if (DISPLAY_VER(dev_priv) >= 7) {
3503 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3504 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3505 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3506 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3507 }
3508
3509 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3510 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3511 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3512 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3513 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3514 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3515
3516 dev_priv->display.wm.hw = *results;
3517}
3518
3519bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3520{
3521 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3522}
3523
3524static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3525 struct intel_wm_config *config)
3526{
3527 struct intel_crtc *crtc;
3528
3529 /* Compute the currently _active_ config */
3530 for_each_intel_crtc(&dev_priv->drm, crtc) {
3531 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3532
3533 if (!wm->pipe_enabled)
3534 continue;
3535
3536 config->sprites_enabled |= wm->sprites_enabled;
3537 config->sprites_scaled |= wm->sprites_scaled;
3538 config->num_pipes_active++;
3539 }
3540}
3541
3542static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3543{
3544 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3545 struct ilk_wm_maximums max;
3546 struct intel_wm_config config = {};
3547 struct ilk_wm_values results = {};
3548 enum intel_ddb_partitioning partitioning;
3549
3550 ilk_compute_wm_config(dev_priv, &config);
3551
3552 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3553 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3554
3555 /* 5/6 split only in single pipe config on IVB+ */
3556 if (DISPLAY_VER(dev_priv) >= 7 &&
3557 config.num_pipes_active == 1 && config.sprites_enabled) {
3558 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3559 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3560
3561 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3562 } else {
3563 best_lp_wm = &lp_wm_1_2;
3564 }
3565
3566 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3567 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3568
3569 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3570
3571 ilk_write_wm_values(dev_priv, &results);
3572}
3573
3574static void ilk_initial_watermarks(struct intel_atomic_state *state,
3575 struct intel_crtc *crtc)
3576{
3577 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3578 const struct intel_crtc_state *crtc_state =
3579 intel_atomic_get_new_crtc_state(state, crtc);
3580
3581 mutex_lock(&dev_priv->display.wm.wm_mutex);
3582 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3583 ilk_program_watermarks(dev_priv);
3584 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3585}
3586
3587static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3588 struct intel_crtc *crtc)
3589{
3590 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3591 const struct intel_crtc_state *crtc_state =
3592 intel_atomic_get_new_crtc_state(state, crtc);
3593
3594 if (!crtc_state->wm.need_postvbl_update)
3595 return;
3596
3597 mutex_lock(&dev_priv->display.wm.wm_mutex);
3598 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3599 ilk_program_watermarks(dev_priv);
3600 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3601}
3602
3603static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3604{
3605 struct drm_device *dev = crtc->base.dev;
3606 struct drm_i915_private *dev_priv = to_i915(dev);
3607 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3608 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3609 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3610 enum pipe pipe = crtc->pipe;
3611
3612 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3613
3614 memset(active, 0, sizeof(*active));
3615
3616 active->pipe_enabled = crtc->active;
3617
3618 if (active->pipe_enabled) {
3619 u32 tmp = hw->wm_pipe[pipe];
3620
3621 /*
3622 * For active pipes LP0 watermark is marked as
3623 * enabled, and LP1+ watermaks as disabled since
3624 * we can't really reverse compute them in case
3625 * multiple pipes are active.
3626 */
3627 active->wm[0].enable = true;
3628 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3629 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3630 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3631 } else {
3632 int level, max_level = ilk_wm_max_level(dev_priv);
3633
3634 /*
3635 * For inactive pipes, all watermark levels
3636 * should be marked as enabled but zeroed,
3637 * which is what we'd compute them to.
3638 */
3639 for (level = 0; level <= max_level; level++)
3640 active->wm[level].enable = true;
3641 }
3642
3643 crtc->wm.active.ilk = *active;
3644}
3645
3646#define _FW_WM(value, plane) \
3647 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3648#define _FW_WM_VLV(value, plane) \
3649 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3650
3651static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3652 struct g4x_wm_values *wm)
3653{
3654 u32 tmp;
3655
3656 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3657 wm->sr.plane = _FW_WM(tmp, SR);
3658 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3659 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3660 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3661
3662 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3663 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3664 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3665 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3666 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3667 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3668 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3669
3670 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3671 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3672 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3673 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3674 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3675}
3676
3677static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3678 struct vlv_wm_values *wm)
3679{
3680 enum pipe pipe;
3681 u32 tmp;
3682
3683 for_each_pipe(dev_priv, pipe) {
3684 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3685
3686 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3687 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3688 wm->ddl[pipe].plane[PLANE_CURSOR] =
3689 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3690 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3691 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3692 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3693 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3694 }
3695
3696 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3697 wm->sr.plane = _FW_WM(tmp, SR);
3698 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3699 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3700 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3701
3702 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3703 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3704 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3705 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3706
3707 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3708 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3709
3710 if (IS_CHERRYVIEW(dev_priv)) {
3711 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3712 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3713 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3714
3715 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3716 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3717 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3718
3719 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3720 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3721 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3722
3723 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3724 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3725 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3726 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3727 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3728 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3729 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3730 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3731 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3732 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3733 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3734 } else {
3735 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3736 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3737 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3738
3739 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3740 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3741 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3742 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3743 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3744 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3745 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3746 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3747 }
3748}
3749
3750#undef _FW_WM
3751#undef _FW_WM_VLV
3752
3753void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3754{
3755 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3756 struct intel_crtc *crtc;
3757
3758 g4x_read_wm_values(dev_priv, wm);
3759
3760 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3761
3762 for_each_intel_crtc(&dev_priv->drm, crtc) {
3763 struct intel_crtc_state *crtc_state =
3764 to_intel_crtc_state(crtc->base.state);
3765 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3766 struct g4x_pipe_wm *raw;
3767 enum pipe pipe = crtc->pipe;
3768 enum plane_id plane_id;
3769 int level, max_level;
3770
3771 active->cxsr = wm->cxsr;
3772 active->hpll_en = wm->hpll_en;
3773 active->fbc_en = wm->fbc_en;
3774
3775 active->sr = wm->sr;
3776 active->hpll = wm->hpll;
3777
3778 for_each_plane_id_on_crtc(crtc, plane_id) {
3779 active->wm.plane[plane_id] =
3780 wm->pipe[pipe].plane[plane_id];
3781 }
3782
3783 if (wm->cxsr && wm->hpll_en)
3784 max_level = G4X_WM_LEVEL_HPLL;
3785 else if (wm->cxsr)
3786 max_level = G4X_WM_LEVEL_SR;
3787 else
3788 max_level = G4X_WM_LEVEL_NORMAL;
3789
3790 level = G4X_WM_LEVEL_NORMAL;
3791 raw = &crtc_state->wm.g4x.raw[level];
3792 for_each_plane_id_on_crtc(crtc, plane_id)
3793 raw->plane[plane_id] = active->wm.plane[plane_id];
3794
3795 level = G4X_WM_LEVEL_SR;
3796 if (level > max_level)
3797 goto out;
3798
3799 raw = &crtc_state->wm.g4x.raw[level];
3800 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3801 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3802 raw->plane[PLANE_SPRITE0] = 0;
3803 raw->fbc = active->sr.fbc;
3804
3805 level = G4X_WM_LEVEL_HPLL;
3806 if (level > max_level)
3807 goto out;
3808
3809 raw = &crtc_state->wm.g4x.raw[level];
3810 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3811 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3812 raw->plane[PLANE_SPRITE0] = 0;
3813 raw->fbc = active->hpll.fbc;
3814
3815 level++;
3816 out:
3817 for_each_plane_id_on_crtc(crtc, plane_id)
3818 g4x_raw_plane_wm_set(crtc_state, level,
3819 plane_id, USHRT_MAX);
3820 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3821
3822 g4x_invalidate_wms(crtc, active, level);
3823
3824 crtc_state->wm.g4x.optimal = *active;
3825 crtc_state->wm.g4x.intermediate = *active;
3826
3827 drm_dbg_kms(&dev_priv->drm,
3828 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3829 pipe_name(pipe),
3830 wm->pipe[pipe].plane[PLANE_PRIMARY],
3831 wm->pipe[pipe].plane[PLANE_CURSOR],
3832 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3833 }
3834
3835 drm_dbg_kms(&dev_priv->drm,
3836 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3837 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3838 drm_dbg_kms(&dev_priv->drm,
3839 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3840 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3841 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3842 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3843 str_yes_no(wm->fbc_en));
3844}
3845
3846void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3847{
3848 struct intel_plane *plane;
3849 struct intel_crtc *crtc;
3850
3851 mutex_lock(&dev_priv->display.wm.wm_mutex);
3852
3853 for_each_intel_plane(&dev_priv->drm, plane) {
3854 struct intel_crtc *crtc =
3855 intel_crtc_for_pipe(dev_priv, plane->pipe);
3856 struct intel_crtc_state *crtc_state =
3857 to_intel_crtc_state(crtc->base.state);
3858 struct intel_plane_state *plane_state =
3859 to_intel_plane_state(plane->base.state);
3860 enum plane_id plane_id = plane->id;
3861 int level, num_levels = intel_wm_num_levels(dev_priv);
3862
3863 if (plane_state->uapi.visible)
3864 continue;
3865
3866 for (level = 0; level < num_levels; level++) {
3867 struct g4x_pipe_wm *raw =
3868 &crtc_state->wm.g4x.raw[level];
3869
3870 raw->plane[plane_id] = 0;
3871
3872 if (plane_id == PLANE_PRIMARY)
3873 raw->fbc = 0;
3874 }
3875 }
3876
3877 for_each_intel_crtc(&dev_priv->drm, crtc) {
3878 struct intel_crtc_state *crtc_state =
3879 to_intel_crtc_state(crtc->base.state);
3880 int ret;
3881
3882 ret = _g4x_compute_pipe_wm(crtc_state);
3883 drm_WARN_ON(&dev_priv->drm, ret);
3884
3885 crtc_state->wm.g4x.intermediate =
3886 crtc_state->wm.g4x.optimal;
3887 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3888 }
3889
3890 g4x_program_watermarks(dev_priv);
3891
3892 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3893}
3894
3895void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3896{
3897 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3898 struct intel_crtc *crtc;
3899 u32 val;
3900
3901 vlv_read_wm_values(dev_priv, wm);
3902
3903 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3904 wm->level = VLV_WM_LEVEL_PM2;
3905
3906 if (IS_CHERRYVIEW(dev_priv)) {
3907 vlv_punit_get(dev_priv);
3908
3909 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3910 if (val & DSP_MAXFIFO_PM5_ENABLE)
3911 wm->level = VLV_WM_LEVEL_PM5;
3912
3913 /*
3914 * If DDR DVFS is disabled in the BIOS, Punit
3915 * will never ack the request. So if that happens
3916 * assume we don't have to enable/disable DDR DVFS
3917 * dynamically. To test that just set the REQ_ACK
3918 * bit to poke the Punit, but don't change the
3919 * HIGH/LOW bits so that we don't actually change
3920 * the current state.
3921 */
3922 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3923 val |= FORCE_DDR_FREQ_REQ_ACK;
3924 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3925
3926 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3927 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3928 drm_dbg_kms(&dev_priv->drm,
3929 "Punit not acking DDR DVFS request, "
3930 "assuming DDR DVFS is disabled\n");
3931 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3932 } else {
3933 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3934 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3935 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3936 }
3937
3938 vlv_punit_put(dev_priv);
3939 }
3940
3941 for_each_intel_crtc(&dev_priv->drm, crtc) {
3942 struct intel_crtc_state *crtc_state =
3943 to_intel_crtc_state(crtc->base.state);
3944 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3945 const struct vlv_fifo_state *fifo_state =
3946 &crtc_state->wm.vlv.fifo_state;
3947 enum pipe pipe = crtc->pipe;
3948 enum plane_id plane_id;
3949 int level;
3950
3951 vlv_get_fifo_size(crtc_state);
3952
3953 active->num_levels = wm->level + 1;
3954 active->cxsr = wm->cxsr;
3955
3956 for (level = 0; level < active->num_levels; level++) {
3957 struct g4x_pipe_wm *raw =
3958 &crtc_state->wm.vlv.raw[level];
3959
3960 active->sr[level].plane = wm->sr.plane;
3961 active->sr[level].cursor = wm->sr.cursor;
3962
3963 for_each_plane_id_on_crtc(crtc, plane_id) {
3964 active->wm[level].plane[plane_id] =
3965 wm->pipe[pipe].plane[plane_id];
3966
3967 raw->plane[plane_id] =
3968 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3969 fifo_state->plane[plane_id]);
3970 }
3971 }
3972
3973 for_each_plane_id_on_crtc(crtc, plane_id)
3974 vlv_raw_plane_wm_set(crtc_state, level,
3975 plane_id, USHRT_MAX);
3976 vlv_invalidate_wms(crtc, active, level);
3977
3978 crtc_state->wm.vlv.optimal = *active;
3979 crtc_state->wm.vlv.intermediate = *active;
3980
3981 drm_dbg_kms(&dev_priv->drm,
3982 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3983 pipe_name(pipe),
3984 wm->pipe[pipe].plane[PLANE_PRIMARY],
3985 wm->pipe[pipe].plane[PLANE_CURSOR],
3986 wm->pipe[pipe].plane[PLANE_SPRITE0],
3987 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3988 }
3989
3990 drm_dbg_kms(&dev_priv->drm,
3991 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3992 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3993}
3994
3995void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3996{
3997 struct intel_plane *plane;
3998 struct intel_crtc *crtc;
3999
4000 mutex_lock(&dev_priv->display.wm.wm_mutex);
4001
4002 for_each_intel_plane(&dev_priv->drm, plane) {
4003 struct intel_crtc *crtc =
4004 intel_crtc_for_pipe(dev_priv, plane->pipe);
4005 struct intel_crtc_state *crtc_state =
4006 to_intel_crtc_state(crtc->base.state);
4007 struct intel_plane_state *plane_state =
4008 to_intel_plane_state(plane->base.state);
4009 enum plane_id plane_id = plane->id;
4010 int level, num_levels = intel_wm_num_levels(dev_priv);
4011
4012 if (plane_state->uapi.visible)
4013 continue;
4014
4015 for (level = 0; level < num_levels; level++) {
4016 struct g4x_pipe_wm *raw =
4017 &crtc_state->wm.vlv.raw[level];
4018
4019 raw->plane[plane_id] = 0;
4020 }
4021 }
4022
4023 for_each_intel_crtc(&dev_priv->drm, crtc) {
4024 struct intel_crtc_state *crtc_state =
4025 to_intel_crtc_state(crtc->base.state);
4026 int ret;
4027
4028 ret = _vlv_compute_pipe_wm(crtc_state);
4029 drm_WARN_ON(&dev_priv->drm, ret);
4030
4031 crtc_state->wm.vlv.intermediate =
4032 crtc_state->wm.vlv.optimal;
4033 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4034 }
4035
4036 vlv_program_watermarks(dev_priv);
4037
4038 mutex_unlock(&dev_priv->display.wm.wm_mutex);
4039}
4040
4041/*
4042 * FIXME should probably kill this and improve
4043 * the real watermark readout/sanitation instead
4044 */
4045static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4046{
4047 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4048 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4049 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4050
4051 /*
4052 * Don't touch WM_LP_SPRITE_ENABLE here.
4053 * Doing so could cause underruns.
4054 */
4055}
4056
4057void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4058{
4059 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4060 struct intel_crtc *crtc;
4061
4062 ilk_init_lp_watermarks(dev_priv);
4063
4064 for_each_intel_crtc(&dev_priv->drm, crtc)
4065 ilk_pipe_wm_get_hw_state(crtc);
4066
4067 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4068 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4069 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4070
4071 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4072 if (DISPLAY_VER(dev_priv) >= 7) {
4073 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4074 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4075 }
4076
4077 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4078 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4079 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4080 else if (IS_IVYBRIDGE(dev_priv))
4081 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4082 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4083
4084 hw->enable_fbc_wm =
4085 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4086}
4087
4088static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4089{
4090 /*
4091 * On Ibex Peak and Cougar Point, we need to disable clock
4092 * gating for the panel power sequencer or it will fail to
4093 * start up when no ports are active.
4094 */
4095 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4096}
4097
4098static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4099{
4100 enum pipe pipe;
4101
4102 for_each_pipe(dev_priv, pipe) {
4103 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4104
4105 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4106 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4107 }
4108}
4109
4110static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4111{
4112 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4113
4114 /*
4115 * Required for FBC
4116 * WaFbcDisableDpfcClockGating:ilk
4117 */
4118 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4119 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4120 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4121
4122 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4123 MARIUNIT_CLOCK_GATE_DISABLE |
4124 SVSMUNIT_CLOCK_GATE_DISABLE);
4125 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4126 VFMUNIT_CLOCK_GATE_DISABLE);
4127
4128 /*
4129 * According to the spec the following bits should be set in
4130 * order to enable memory self-refresh
4131 * The bit 22/21 of 0x42004
4132 * The bit 5 of 0x42020
4133 * The bit 15 of 0x45000
4134 */
4135 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4136 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4137 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4138 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4139 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4140 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4141 DISP_FBC_WM_DIS));
4142
4143 /*
4144 * Based on the document from hardware guys the following bits
4145 * should be set unconditionally in order to enable FBC.
4146 * The bit 22 of 0x42000
4147 * The bit 22 of 0x42004
4148 * The bit 7,8,9 of 0x42020.
4149 */
4150 if (IS_IRONLAKE_M(dev_priv)) {
4151 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4152 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4153 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4154 }
4155
4156 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4157
4158 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4159
4160 g4x_disable_trickle_feed(dev_priv);
4161
4162 ibx_init_clock_gating(dev_priv);
4163}
4164
4165static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4166{
4167 enum pipe pipe;
4168 u32 val;
4169
4170 /*
4171 * On Ibex Peak and Cougar Point, we need to disable clock
4172 * gating for the panel power sequencer or it will fail to
4173 * start up when no ports are active.
4174 */
4175 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4176 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4177 PCH_CPUNIT_CLOCK_GATE_DISABLE);
4178 intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4179 /* The below fixes the weird display corruption, a few pixels shifted
4180 * downward, on (only) LVDS of some HP laptops with IVY.
4181 */
4182 for_each_pipe(dev_priv, pipe) {
4183 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4184 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4185 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4186 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4187 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4188 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4189 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4190 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4191 }
4192 /* WADP0ClockGatingDisable */
4193 for_each_pipe(dev_priv, pipe) {
4194 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4195 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4196 }
4197}
4198
4199static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4200{
4201 u32 tmp;
4202
4203 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4204 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4205 drm_dbg_kms(&dev_priv->drm,
4206 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4207 tmp);
4208}
4209
4210static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4211{
4212 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4213
4214 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4215
4216 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4217
4218 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4219 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4220 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4221 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4222
4223 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4224 * gating disable must be set. Failure to set it results in
4225 * flickering pixels due to Z write ordering failures after
4226 * some amount of runtime in the Mesa "fire" demo, and Unigine
4227 * Sanctuary and Tropics, and apparently anything else with
4228 * alpha test or pixel discard.
4229 *
4230 * According to the spec, bit 11 (RCCUNIT) must also be set,
4231 * but we didn't debug actual testcases to find it out.
4232 *
4233 * WaDisableRCCUnitClockGating:snb
4234 * WaDisableRCPBUnitClockGating:snb
4235 */
4236 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4237 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4238 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4239
4240 /*
4241 * According to the spec the following bits should be
4242 * set in order to enable memory self-refresh and fbc:
4243 * The bit21 and bit22 of 0x42000
4244 * The bit21 and bit22 of 0x42004
4245 * The bit5 and bit7 of 0x42020
4246 * The bit14 of 0x70180
4247 * The bit14 of 0x71180
4248 *
4249 * WaFbcAsynchFlipDisableFbcQueue:snb
4250 */
4251 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4252 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4253 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4254 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4255 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4256 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4257 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4258 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4259 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4260 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4261
4262 g4x_disable_trickle_feed(dev_priv);
4263
4264 cpt_init_clock_gating(dev_priv);
4265
4266 gen6_check_mch_setup(dev_priv);
4267}
4268
4269static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4270{
4271 /*
4272 * TODO: this bit should only be enabled when really needed, then
4273 * disabled when not needed anymore in order to save power.
4274 */
4275 if (HAS_PCH_LPT_LP(dev_priv))
4276 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4277 0, PCH_LP_PARTITION_LEVEL_DISABLE);
4278
4279 /* WADPOClockGatingDisable:hsw */
4280 intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4281 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4282}
4283
4284static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4285{
4286 if (HAS_PCH_LPT_LP(dev_priv)) {
4287 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4288
4289 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4290 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4291 }
4292}
4293
4294static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4295 int general_prio_credits,
4296 int high_prio_credits)
4297{
4298 u32 misccpctl;
4299 u32 val;
4300
4301 /* WaTempDisableDOPClkGating:bdw */
4302 misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4303 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4304
4305 val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4306 val &= ~L3_PRIO_CREDITS_MASK;
4307 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4308 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4309 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4310
4311 /*
4312 * Wait at least 100 clocks before re-enabling clock gating.
4313 * See the definition of L3SQCREG1 in BSpec.
4314 */
4315 intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4316 udelay(1);
4317 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
4318}
4319
4320static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4321{
4322 /* Wa_1409120013:icl,ehl */
4323 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4324 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4325
4326 /*Wa_14010594013:icl, ehl */
4327 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4328 0, ICL_DELAY_PMRSP);
4329}
4330
4331static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4332{
4333 /* Wa_1409120013 */
4334 if (DISPLAY_VER(dev_priv) == 12)
4335 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4336 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4337
4338 /* Wa_1409825376:tgl (pre-prod)*/
4339 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4340 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4341
4342 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4343 if (DISPLAY_VER(dev_priv) == 12)
4344 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4345 CLKREQ_POLICY_MEM_UP_OVRD, 0);
4346}
4347
4348static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4349{
4350 gen12lp_init_clock_gating(dev_priv);
4351
4352 /* Wa_22011091694:adlp */
4353 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4354
4355 /* Bspec/49189 Initialize Sequence */
4356 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4357}
4358
4359static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4360{
4361 gen12lp_init_clock_gating(dev_priv);
4362
4363 /* Wa_1409836686:dg1[a0] */
4364 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4365 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4366}
4367
4368static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4369{
4370 /* Wa_22010146351:xehpsdv */
4371 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4372 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4373}
4374
4375static void dg2_init_clock_gating(struct drm_i915_private *i915)
4376{
4377 /* Wa_22010954014:dg2 */
4378 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4379 SGSI_SIDECLK_DIS);
4380
4381 /*
4382 * Wa_14010733611:dg2_g10
4383 * Wa_22010146351:dg2_g10
4384 */
4385 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4386 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4387 SGR_DIS | SGGI_DIS);
4388}
4389
4390static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4391{
4392 /* Wa_14012385139:pvc */
4393 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4394 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4395
4396 /* Wa_22010954014:pvc */
4397 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4398 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4399}
4400
4401static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4402{
4403 if (!HAS_PCH_CNP(dev_priv))
4404 return;
4405
4406 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4407 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4408}
4409
4410static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4411{
4412 cnp_init_clock_gating(dev_priv);
4413 gen9_init_clock_gating(dev_priv);
4414
4415 /* WAC6entrylatency:cfl */
4416 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4417
4418 /*
4419 * WaFbcTurnOffFbcWatermark:cfl
4420 * Display WA #0562: cfl
4421 */
4422 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4423
4424 /*
4425 * WaFbcNukeOnHostModify:cfl
4426 * Display WA #0873: cfl
4427 */
4428 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4429 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4430}
4431
4432static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4433{
4434 gen9_init_clock_gating(dev_priv);
4435
4436 /* WAC6entrylatency:kbl */
4437 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4438
4439 /* WaDisableSDEUnitClockGating:kbl */
4440 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4441 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4442 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4443
4444 /* WaDisableGamClockGating:kbl */
4445 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4446 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4447 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4448
4449 /*
4450 * WaFbcTurnOffFbcWatermark:kbl
4451 * Display WA #0562: kbl
4452 */
4453 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4454
4455 /*
4456 * WaFbcNukeOnHostModify:kbl
4457 * Display WA #0873: kbl
4458 */
4459 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4460 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4461}
4462
4463static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4464{
4465 gen9_init_clock_gating(dev_priv);
4466
4467 /* WaDisableDopClockGating:skl */
4468 intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4469 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4470
4471 /* WAC6entrylatency:skl */
4472 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4473
4474 /*
4475 * WaFbcTurnOffFbcWatermark:skl
4476 * Display WA #0562: skl
4477 */
4478 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4479
4480 /*
4481 * WaFbcNukeOnHostModify:skl
4482 * Display WA #0873: skl
4483 */
4484 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4485 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4486
4487 /*
4488 * WaFbcHighMemBwCorruptionAvoidance:skl
4489 * Display WA #0883: skl
4490 */
4491 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4492}
4493
4494static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4495{
4496 enum pipe pipe;
4497
4498 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4499 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4500
4501 /* WaSwitchSolVfFArbitrationPriority:bdw */
4502 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4503
4504 /* WaPsrDPAMaskVBlankInSRD:bdw */
4505 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4506
4507 for_each_pipe(dev_priv, pipe) {
4508 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4509 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4510 0, BDW_DPRS_MASK_VBLANK_SRD);
4511 }
4512
4513 /* WaVSRefCountFullforceMissDisable:bdw */
4514 /* WaDSRefCountFullforceMissDisable:bdw */
4515 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4516 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4517
4518 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4519 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4520
4521 /* WaDisableSDEUnitClockGating:bdw */
4522 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4523
4524 /* WaProgramL3SqcReg1Default:bdw */
4525 gen8_set_l3sqc_credits(dev_priv, 30, 2);
4526
4527 /* WaKVMNotificationOnConfigChange:bdw */
4528 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4529 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4530
4531 lpt_init_clock_gating(dev_priv);
4532
4533 /* WaDisableDopClockGating:bdw
4534 *
4535 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4536 * clock gating.
4537 */
4538 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4539}
4540
4541static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4542{
4543 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4544 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4545
4546 /* This is required by WaCatErrorRejectionIssue:hsw */
4547 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4548 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4549
4550 /* WaSwitchSolVfFArbitrationPriority:hsw */
4551 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4552
4553 lpt_init_clock_gating(dev_priv);
4554}
4555
4556static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4557{
4558 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4559
4560 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4561 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4562
4563 /* WaDisableBackToBackFlipFix:ivb */
4564 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4565 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4566 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4567
4568 if (IS_IVB_GT1(dev_priv))
4569 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4570 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4571 else {
4572 /* must write both registers */
4573 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4574 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4575 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4576 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4577 }
4578
4579 /*
4580 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4581 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4582 */
4583 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4584 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4585
4586 /* This is required by WaCatErrorRejectionIssue:ivb */
4587 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4588 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4589
4590 g4x_disable_trickle_feed(dev_priv);
4591
4592 intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4593 GEN6_MBC_SNPCR_MED);
4594
4595 if (!HAS_PCH_NOP(dev_priv))
4596 cpt_init_clock_gating(dev_priv);
4597
4598 gen6_check_mch_setup(dev_priv);
4599}
4600
4601static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4602{
4603 /* WaDisableBackToBackFlipFix:vlv */
4604 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4605 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4606 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4607
4608 /* WaDisableDopClockGating:vlv */
4609 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4610 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4611
4612 /* This is required by WaCatErrorRejectionIssue:vlv */
4613 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4614 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4615
4616 /*
4617 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4618 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4619 */
4620 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4621 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4622
4623 /* WaDisableL3Bank2xClockGate:vlv
4624 * Disabling L3 clock gating- MMIO 940c[25] = 1
4625 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4626 intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4627
4628 /*
4629 * WaDisableVLVClockGating_VBIIssue:vlv
4630 * Disable clock gating on th GCFG unit to prevent a delay
4631 * in the reporting of vblank events.
4632 */
4633 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4634}
4635
4636static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4637{
4638 /* WaVSRefCountFullforceMissDisable:chv */
4639 /* WaDSRefCountFullforceMissDisable:chv */
4640 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4641 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4642
4643 /* WaDisableSemaphoreAndSyncFlipWait:chv */
4644 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4645 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4646
4647 /* WaDisableCSUnitClockGating:chv */
4648 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4649
4650 /* WaDisableSDEUnitClockGating:chv */
4651 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4652
4653 /*
4654 * WaProgramL3SqcReg1Default:chv
4655 * See gfxspecs/Related Documents/Performance Guide/
4656 * LSQC Setting Recommendations.
4657 */
4658 gen8_set_l3sqc_credits(dev_priv, 38, 2);
4659}
4660
4661static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4662{
4663 u32 dspclk_gate;
4664
4665 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4666 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4667 GS_UNIT_CLOCK_GATE_DISABLE |
4668 CL_UNIT_CLOCK_GATE_DISABLE);
4669 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4670 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4671 OVRUNIT_CLOCK_GATE_DISABLE |
4672 OVCUNIT_CLOCK_GATE_DISABLE;
4673 if (IS_GM45(dev_priv))
4674 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4675 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4676
4677 g4x_disable_trickle_feed(dev_priv);
4678}
4679
4680static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4681{
4682 struct intel_uncore *uncore = &dev_priv->uncore;
4683
4684 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4685 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4686 intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4687 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4688 intel_uncore_write16(uncore, DEUC, 0);
4689 intel_uncore_write(uncore,
4690 MI_ARB_STATE,
4691 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4692}
4693
4694static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4695{
4696 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4697 I965_RCC_CLOCK_GATE_DISABLE |
4698 I965_RCPB_CLOCK_GATE_DISABLE |
4699 I965_ISC_CLOCK_GATE_DISABLE |
4700 I965_FBC_CLOCK_GATE_DISABLE);
4701 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4702 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4703 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4704}
4705
4706static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4707{
4708 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4709
4710 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4711 DSTATE_DOT_CLOCK_GATING;
4712 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4713
4714 if (IS_PINEVIEW(dev_priv))
4715 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4716 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4717
4718 /* IIR "flip pending" means done if this bit is set */
4719 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4720 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4721
4722 /* interrupts should cause a wake up from C3 */
4723 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4724
4725 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4726 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4727
4728 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4729 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4730}
4731
4732static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4733{
4734 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4735
4736 /* interrupts should cause a wake up from C3 */
4737 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4738 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4739
4740 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4741 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4742
4743 /*
4744 * Have FBC ignore 3D activity since we use software
4745 * render tracking, and otherwise a pure 3D workload
4746 * (even if it just renders a single frame and then does
4747 * abosultely nothing) would not allow FBC to recompress
4748 * until a 2D blit occurs.
4749 */
4750 intel_uncore_write(&dev_priv->uncore, SCPD0,
4751 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4752}
4753
4754static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4755{
4756 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4757 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4758 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4759}
4760
4761void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4762{
4763 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4764}
4765
4766void intel_suspend_hw(struct drm_i915_private *dev_priv)
4767{
4768 if (HAS_PCH_LPT(dev_priv))
4769 lpt_suspend_hw(dev_priv);
4770}
4771
4772static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4773{
4774 drm_dbg_kms(&dev_priv->drm,
4775 "No clock gating settings or workarounds applied.\n");
4776}
4777
4778#define CG_FUNCS(platform) \
4779static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4780 .init_clock_gating = platform##_init_clock_gating, \
4781}
4782
4783CG_FUNCS(pvc);
4784CG_FUNCS(dg2);
4785CG_FUNCS(xehpsdv);
4786CG_FUNCS(adlp);
4787CG_FUNCS(dg1);
4788CG_FUNCS(gen12lp);
4789CG_FUNCS(icl);
4790CG_FUNCS(cfl);
4791CG_FUNCS(skl);
4792CG_FUNCS(kbl);
4793CG_FUNCS(bxt);
4794CG_FUNCS(glk);
4795CG_FUNCS(bdw);
4796CG_FUNCS(chv);
4797CG_FUNCS(hsw);
4798CG_FUNCS(ivb);
4799CG_FUNCS(vlv);
4800CG_FUNCS(gen6);
4801CG_FUNCS(ilk);
4802CG_FUNCS(g4x);
4803CG_FUNCS(i965gm);
4804CG_FUNCS(i965g);
4805CG_FUNCS(gen3);
4806CG_FUNCS(i85x);
4807CG_FUNCS(i830);
4808CG_FUNCS(nop);
4809#undef CG_FUNCS
4810
4811/**
4812 * intel_init_clock_gating_hooks - setup the clock gating hooks
4813 * @dev_priv: device private
4814 *
4815 * Setup the hooks that configure which clocks of a given platform can be
4816 * gated and also apply various GT and display specific workarounds for these
4817 * platforms. Note that some GT specific workarounds are applied separately
4818 * when GPU contexts or batchbuffers start their execution.
4819 */
4820void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4821{
4822 if (IS_PONTEVECCHIO(dev_priv))
4823 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4824 else if (IS_DG2(dev_priv))
4825 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4826 else if (IS_XEHPSDV(dev_priv))
4827 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4828 else if (IS_ALDERLAKE_P(dev_priv))
4829 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4830 else if (IS_DG1(dev_priv))
4831 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4832 else if (GRAPHICS_VER(dev_priv) == 12)
4833 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4834 else if (GRAPHICS_VER(dev_priv) == 11)
4835 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4836 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4837 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4838 else if (IS_SKYLAKE(dev_priv))
4839 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4840 else if (IS_KABYLAKE(dev_priv))
4841 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4842 else if (IS_BROXTON(dev_priv))
4843 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4844 else if (IS_GEMINILAKE(dev_priv))
4845 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4846 else if (IS_BROADWELL(dev_priv))
4847 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4848 else if (IS_CHERRYVIEW(dev_priv))
4849 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4850 else if (IS_HASWELL(dev_priv))
4851 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4852 else if (IS_IVYBRIDGE(dev_priv))
4853 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4854 else if (IS_VALLEYVIEW(dev_priv))
4855 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4856 else if (GRAPHICS_VER(dev_priv) == 6)
4857 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4858 else if (GRAPHICS_VER(dev_priv) == 5)
4859 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4860 else if (IS_G4X(dev_priv))
4861 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4862 else if (IS_I965GM(dev_priv))
4863 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4864 else if (IS_I965G(dev_priv))
4865 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4866 else if (GRAPHICS_VER(dev_priv) == 3)
4867 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4868 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4869 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4870 else if (GRAPHICS_VER(dev_priv) == 2)
4871 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4872 else {
4873 MISSING_CASE(INTEL_DEVID(dev_priv));
4874 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4875 }
4876}
4877
4878static const struct intel_wm_funcs ilk_wm_funcs = {
4879 .compute_pipe_wm = ilk_compute_pipe_wm,
4880 .compute_intermediate_wm = ilk_compute_intermediate_wm,
4881 .initial_watermarks = ilk_initial_watermarks,
4882 .optimize_watermarks = ilk_optimize_watermarks,
4883};
4884
4885static const struct intel_wm_funcs vlv_wm_funcs = {
4886 .compute_pipe_wm = vlv_compute_pipe_wm,
4887 .compute_intermediate_wm = vlv_compute_intermediate_wm,
4888 .initial_watermarks = vlv_initial_watermarks,
4889 .optimize_watermarks = vlv_optimize_watermarks,
4890 .atomic_update_watermarks = vlv_atomic_update_fifo,
4891};
4892
4893static const struct intel_wm_funcs g4x_wm_funcs = {
4894 .compute_pipe_wm = g4x_compute_pipe_wm,
4895 .compute_intermediate_wm = g4x_compute_intermediate_wm,
4896 .initial_watermarks = g4x_initial_watermarks,
4897 .optimize_watermarks = g4x_optimize_watermarks,
4898};
4899
4900static const struct intel_wm_funcs pnv_wm_funcs = {
4901 .update_wm = pnv_update_wm,
4902};
4903
4904static const struct intel_wm_funcs i965_wm_funcs = {
4905 .update_wm = i965_update_wm,
4906};
4907
4908static const struct intel_wm_funcs i9xx_wm_funcs = {
4909 .update_wm = i9xx_update_wm,
4910};
4911
4912static const struct intel_wm_funcs i845_wm_funcs = {
4913 .update_wm = i845_update_wm,
4914};
4915
4916static const struct intel_wm_funcs nop_funcs = {
4917};
4918
4919/* Set up chip specific power management-related functions */
4920void intel_init_pm(struct drm_i915_private *dev_priv)
4921{
4922 if (DISPLAY_VER(dev_priv) >= 9) {
4923 skl_wm_init(dev_priv);
4924 return;
4925 }
4926
4927 /* For cxsr */
4928 if (IS_PINEVIEW(dev_priv))
4929 pnv_get_mem_freq(dev_priv);
4930 else if (GRAPHICS_VER(dev_priv) == 5)
4931 ilk_get_mem_freq(dev_priv);
4932
4933 /* For FIFO watermark updates */
4934 if (HAS_PCH_SPLIT(dev_priv)) {
4935 ilk_setup_wm_latency(dev_priv);
4936
4937 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4938 dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4939 (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4940 dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4941 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4942 } else {
4943 drm_dbg_kms(&dev_priv->drm,
4944 "Failed to read display plane latency. "
4945 "Disable CxSR\n");
4946 dev_priv->display.funcs.wm = &nop_funcs;
4947 }
4948 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4949 vlv_setup_wm_latency(dev_priv);
4950 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4951 } else if (IS_G4X(dev_priv)) {
4952 g4x_setup_wm_latency(dev_priv);
4953 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4954 } else if (IS_PINEVIEW(dev_priv)) {
4955 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4956 dev_priv->is_ddr3,
4957 dev_priv->fsb_freq,
4958 dev_priv->mem_freq)) {
4959 drm_info(&dev_priv->drm,
4960 "failed to find known CxSR latency "
4961 "(found ddr%s fsb freq %d, mem freq %d), "
4962 "disabling CxSR\n",
4963 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4964 dev_priv->fsb_freq, dev_priv->mem_freq);
4965 /* Disable CxSR and never update its watermark again */
4966 intel_set_memory_cxsr(dev_priv, false);
4967 dev_priv->display.funcs.wm = &nop_funcs;
4968 } else
4969 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4970 } else if (DISPLAY_VER(dev_priv) == 4) {
4971 dev_priv->display.funcs.wm = &i965_wm_funcs;
4972 } else if (DISPLAY_VER(dev_priv) == 3) {
4973 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4974 } else if (DISPLAY_VER(dev_priv) == 2) {
4975 if (INTEL_NUM_PIPES(dev_priv) == 1)
4976 dev_priv->display.funcs.wm = &i845_wm_funcs;
4977 else
4978 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4979 } else {
4980 drm_err(&dev_priv->drm,
4981 "unexpected fall-through in %s\n", __func__);
4982 dev_priv->display.funcs.wm = &nop_funcs;
4983 }
4984}
4985
4986void intel_pm_setup(struct drm_i915_private *dev_priv)
4987{
4988 dev_priv->runtime_pm.suspended = false;
4989 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
4990}
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include <linux/module.h>
29#include <linux/pm_runtime.h>
30
31#include <drm/drm_atomic_helper.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_plane_helper.h>
34
35#include "display/intel_atomic.h"
36#include "display/intel_bw.h"
37#include "display/intel_display_types.h"
38#include "display/intel_fbc.h"
39#include "display/intel_sprite.h"
40
41#include "gt/intel_llc.h"
42
43#include "i915_drv.h"
44#include "i915_fixed.h"
45#include "i915_irq.h"
46#include "i915_trace.h"
47#include "intel_pm.h"
48#include "intel_sideband.h"
49#include "../../../platform/x86/intel_ips.h"
50
51/* Stores plane specific WM parameters */
52struct skl_wm_params {
53 bool x_tiled, y_tiled;
54 bool rc_surface;
55 bool is_planar;
56 u32 width;
57 u8 cpp;
58 u32 plane_pixel_rate;
59 u32 y_min_scanlines;
60 u32 plane_bytes_per_line;
61 uint_fixed_16_16_t plane_blocks_per_line;
62 uint_fixed_16_16_t y_tile_minimum;
63 u32 linetime_us;
64 u32 dbuf_block_size;
65};
66
67/* used in computing the new watermarks state */
68struct intel_wm_config {
69 unsigned int num_pipes_active;
70 bool sprites_enabled;
71 bool sprites_scaled;
72};
73
74static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
75{
76 if (HAS_LLC(dev_priv)) {
77 /*
78 * WaCompressedResourceDisplayNewHashMode:skl,kbl
79 * Display WA #0390: skl,kbl
80 *
81 * Must match Sampler, Pixel Back End, and Media. See
82 * WaCompressedResourceSamplerPbeMediaNewHashMode.
83 */
84 I915_WRITE(CHICKEN_PAR1_1,
85 I915_READ(CHICKEN_PAR1_1) |
86 SKL_DE_COMPRESSED_HASH_MODE);
87 }
88
89 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
90 I915_WRITE(CHICKEN_PAR1_1,
91 I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
92
93 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
94 I915_WRITE(GEN8_CHICKEN_DCPR_1,
95 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
96
97 /*
98 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
99 * Display WA #0859: skl,bxt,kbl,glk,cfl
100 */
101 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
102 DISP_FBC_MEMORY_WAKE);
103
104 if (IS_SKYLAKE(dev_priv)) {
105 /* WaDisableDopClockGating */
106 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
107 & ~GEN7_DOP_CLOCK_GATE_ENABLE);
108 }
109}
110
111static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
112{
113 gen9_init_clock_gating(dev_priv);
114
115 /* WaDisableSDEUnitClockGating:bxt */
116 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
117 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
118
119 /*
120 * FIXME:
121 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
122 */
123 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
124 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
125
126 /*
127 * Wa: Backlight PWM may stop in the asserted state, causing backlight
128 * to stay fully on.
129 */
130 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
131 PWM1_GATING_DIS | PWM2_GATING_DIS);
132
133 /*
134 * Lower the display internal timeout.
135 * This is needed to avoid any hard hangs when DSI port PLL
136 * is off and a MMIO access is attempted by any privilege
137 * application, using batch buffers or any other means.
138 */
139 I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
140
141 /*
142 * WaFbcTurnOffFbcWatermark:bxt
143 * Display WA #0562: bxt
144 */
145 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
146 DISP_FBC_WM_DIS);
147
148 /*
149 * WaFbcHighMemBwCorruptionAvoidance:bxt
150 * Display WA #0883: bxt
151 */
152 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
153 ILK_DPFC_DISABLE_DUMMY0);
154}
155
156static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
157{
158 gen9_init_clock_gating(dev_priv);
159
160 /*
161 * WaDisablePWMClockGating:glk
162 * Backlight PWM may stop in the asserted state, causing backlight
163 * to stay fully on.
164 */
165 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
166 PWM1_GATING_DIS | PWM2_GATING_DIS);
167}
168
169static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
170{
171 u32 tmp;
172
173 tmp = I915_READ(CLKCFG);
174
175 switch (tmp & CLKCFG_FSB_MASK) {
176 case CLKCFG_FSB_533:
177 dev_priv->fsb_freq = 533; /* 133*4 */
178 break;
179 case CLKCFG_FSB_800:
180 dev_priv->fsb_freq = 800; /* 200*4 */
181 break;
182 case CLKCFG_FSB_667:
183 dev_priv->fsb_freq = 667; /* 167*4 */
184 break;
185 case CLKCFG_FSB_400:
186 dev_priv->fsb_freq = 400; /* 100*4 */
187 break;
188 }
189
190 switch (tmp & CLKCFG_MEM_MASK) {
191 case CLKCFG_MEM_533:
192 dev_priv->mem_freq = 533;
193 break;
194 case CLKCFG_MEM_667:
195 dev_priv->mem_freq = 667;
196 break;
197 case CLKCFG_MEM_800:
198 dev_priv->mem_freq = 800;
199 break;
200 }
201
202 /* detect pineview DDR3 setting */
203 tmp = I915_READ(CSHRDDR3CTL);
204 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
205}
206
207static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
208{
209 u16 ddrpll, csipll;
210
211 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
212 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
213
214 switch (ddrpll & 0xff) {
215 case 0xc:
216 dev_priv->mem_freq = 800;
217 break;
218 case 0x10:
219 dev_priv->mem_freq = 1066;
220 break;
221 case 0x14:
222 dev_priv->mem_freq = 1333;
223 break;
224 case 0x18:
225 dev_priv->mem_freq = 1600;
226 break;
227 default:
228 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
229 ddrpll & 0xff);
230 dev_priv->mem_freq = 0;
231 break;
232 }
233
234 switch (csipll & 0x3ff) {
235 case 0x00c:
236 dev_priv->fsb_freq = 3200;
237 break;
238 case 0x00e:
239 dev_priv->fsb_freq = 3733;
240 break;
241 case 0x010:
242 dev_priv->fsb_freq = 4266;
243 break;
244 case 0x012:
245 dev_priv->fsb_freq = 4800;
246 break;
247 case 0x014:
248 dev_priv->fsb_freq = 5333;
249 break;
250 case 0x016:
251 dev_priv->fsb_freq = 5866;
252 break;
253 case 0x018:
254 dev_priv->fsb_freq = 6400;
255 break;
256 default:
257 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
258 csipll & 0x3ff);
259 dev_priv->fsb_freq = 0;
260 break;
261 }
262}
263
264static const struct cxsr_latency cxsr_latency_table[] = {
265 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
266 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
267 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
268 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
269 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
270
271 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
272 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
273 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
274 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
275 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
276
277 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
278 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
279 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
280 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
281 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
282
283 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
284 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
285 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
286 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
287 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
288
289 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
290 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
291 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
292 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
293 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
294
295 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
296 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
297 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
298 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
299 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
300};
301
302static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
303 bool is_ddr3,
304 int fsb,
305 int mem)
306{
307 const struct cxsr_latency *latency;
308 int i;
309
310 if (fsb == 0 || mem == 0)
311 return NULL;
312
313 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
314 latency = &cxsr_latency_table[i];
315 if (is_desktop == latency->is_desktop &&
316 is_ddr3 == latency->is_ddr3 &&
317 fsb == latency->fsb_freq && mem == latency->mem_freq)
318 return latency;
319 }
320
321 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
322
323 return NULL;
324}
325
326static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
327{
328 u32 val;
329
330 vlv_punit_get(dev_priv);
331
332 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
333 if (enable)
334 val &= ~FORCE_DDR_HIGH_FREQ;
335 else
336 val |= FORCE_DDR_HIGH_FREQ;
337 val &= ~FORCE_DDR_LOW_FREQ;
338 val |= FORCE_DDR_FREQ_REQ_ACK;
339 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
340
341 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
342 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
343 drm_err(&dev_priv->drm,
344 "timed out waiting for Punit DDR DVFS request\n");
345
346 vlv_punit_put(dev_priv);
347}
348
349static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
350{
351 u32 val;
352
353 vlv_punit_get(dev_priv);
354
355 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
356 if (enable)
357 val |= DSP_MAXFIFO_PM5_ENABLE;
358 else
359 val &= ~DSP_MAXFIFO_PM5_ENABLE;
360 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
361
362 vlv_punit_put(dev_priv);
363}
364
365#define FW_WM(value, plane) \
366 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
367
368static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
369{
370 bool was_enabled;
371 u32 val;
372
373 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
374 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
375 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
376 POSTING_READ(FW_BLC_SELF_VLV);
377 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
378 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
379 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
380 POSTING_READ(FW_BLC_SELF);
381 } else if (IS_PINEVIEW(dev_priv)) {
382 val = I915_READ(DSPFW3);
383 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
384 if (enable)
385 val |= PINEVIEW_SELF_REFRESH_EN;
386 else
387 val &= ~PINEVIEW_SELF_REFRESH_EN;
388 I915_WRITE(DSPFW3, val);
389 POSTING_READ(DSPFW3);
390 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
391 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
392 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
393 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
394 I915_WRITE(FW_BLC_SELF, val);
395 POSTING_READ(FW_BLC_SELF);
396 } else if (IS_I915GM(dev_priv)) {
397 /*
398 * FIXME can't find a bit like this for 915G, and
399 * and yet it does have the related watermark in
400 * FW_BLC_SELF. What's going on?
401 */
402 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
403 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
404 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
405 I915_WRITE(INSTPM, val);
406 POSTING_READ(INSTPM);
407 } else {
408 return false;
409 }
410
411 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
412
413 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
414 enableddisabled(enable),
415 enableddisabled(was_enabled));
416
417 return was_enabled;
418}
419
420/**
421 * intel_set_memory_cxsr - Configure CxSR state
422 * @dev_priv: i915 device
423 * @enable: Allow vs. disallow CxSR
424 *
425 * Allow or disallow the system to enter a special CxSR
426 * (C-state self refresh) state. What typically happens in CxSR mode
427 * is that several display FIFOs may get combined into a single larger
428 * FIFO for a particular plane (so called max FIFO mode) to allow the
429 * system to defer memory fetches longer, and the memory will enter
430 * self refresh.
431 *
432 * Note that enabling CxSR does not guarantee that the system enter
433 * this special mode, nor does it guarantee that the system stays
434 * in that mode once entered. So this just allows/disallows the system
435 * to autonomously utilize the CxSR mode. Other factors such as core
436 * C-states will affect when/if the system actually enters/exits the
437 * CxSR mode.
438 *
439 * Note that on VLV/CHV this actually only controls the max FIFO mode,
440 * and the system is free to enter/exit memory self refresh at any time
441 * even when the use of CxSR has been disallowed.
442 *
443 * While the system is actually in the CxSR/max FIFO mode, some plane
444 * control registers will not get latched on vblank. Thus in order to
445 * guarantee the system will respond to changes in the plane registers
446 * we must always disallow CxSR prior to making changes to those registers.
447 * Unfortunately the system will re-evaluate the CxSR conditions at
448 * frame start which happens after vblank start (which is when the plane
449 * registers would get latched), so we can't proceed with the plane update
450 * during the same frame where we disallowed CxSR.
451 *
452 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
453 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
454 * the hardware w.r.t. HPLL SR when writing to plane registers.
455 * Disallowing just CxSR is sufficient.
456 */
457bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
458{
459 bool ret;
460
461 mutex_lock(&dev_priv->wm.wm_mutex);
462 ret = _intel_set_memory_cxsr(dev_priv, enable);
463 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
464 dev_priv->wm.vlv.cxsr = enable;
465 else if (IS_G4X(dev_priv))
466 dev_priv->wm.g4x.cxsr = enable;
467 mutex_unlock(&dev_priv->wm.wm_mutex);
468
469 return ret;
470}
471
472/*
473 * Latency for FIFO fetches is dependent on several factors:
474 * - memory configuration (speed, channels)
475 * - chipset
476 * - current MCH state
477 * It can be fairly high in some situations, so here we assume a fairly
478 * pessimal value. It's a tradeoff between extra memory fetches (if we
479 * set this value too high, the FIFO will fetch frequently to stay full)
480 * and power consumption (set it too low to save power and we might see
481 * FIFO underruns and display "flicker").
482 *
483 * A value of 5us seems to be a good balance; safe for very low end
484 * platforms but not overly aggressive on lower latency configs.
485 */
486static const int pessimal_latency_ns = 5000;
487
488#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
489 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
490
491static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
492{
493 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
494 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
495 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
496 enum pipe pipe = crtc->pipe;
497 int sprite0_start, sprite1_start;
498 u32 dsparb, dsparb2, dsparb3;
499
500 switch (pipe) {
501 case PIPE_A:
502 dsparb = I915_READ(DSPARB);
503 dsparb2 = I915_READ(DSPARB2);
504 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
505 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
506 break;
507 case PIPE_B:
508 dsparb = I915_READ(DSPARB);
509 dsparb2 = I915_READ(DSPARB2);
510 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
511 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
512 break;
513 case PIPE_C:
514 dsparb2 = I915_READ(DSPARB2);
515 dsparb3 = I915_READ(DSPARB3);
516 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
517 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
518 break;
519 default:
520 MISSING_CASE(pipe);
521 return;
522 }
523
524 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
525 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
526 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
527 fifo_state->plane[PLANE_CURSOR] = 63;
528}
529
530static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
531 enum i9xx_plane_id i9xx_plane)
532{
533 u32 dsparb = I915_READ(DSPARB);
534 int size;
535
536 size = dsparb & 0x7f;
537 if (i9xx_plane == PLANE_B)
538 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
539
540 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
541 dsparb, plane_name(i9xx_plane), size);
542
543 return size;
544}
545
546static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
547 enum i9xx_plane_id i9xx_plane)
548{
549 u32 dsparb = I915_READ(DSPARB);
550 int size;
551
552 size = dsparb & 0x1ff;
553 if (i9xx_plane == PLANE_B)
554 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
555 size >>= 1; /* Convert to cachelines */
556
557 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
558 dsparb, plane_name(i9xx_plane), size);
559
560 return size;
561}
562
563static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
564 enum i9xx_plane_id i9xx_plane)
565{
566 u32 dsparb = I915_READ(DSPARB);
567 int size;
568
569 size = dsparb & 0x7f;
570 size >>= 2; /* Convert to cachelines */
571
572 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
573 dsparb, plane_name(i9xx_plane), size);
574
575 return size;
576}
577
578/* Pineview has different values for various configs */
579static const struct intel_watermark_params pnv_display_wm = {
580 .fifo_size = PINEVIEW_DISPLAY_FIFO,
581 .max_wm = PINEVIEW_MAX_WM,
582 .default_wm = PINEVIEW_DFT_WM,
583 .guard_size = PINEVIEW_GUARD_WM,
584 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
585};
586
587static const struct intel_watermark_params pnv_display_hplloff_wm = {
588 .fifo_size = PINEVIEW_DISPLAY_FIFO,
589 .max_wm = PINEVIEW_MAX_WM,
590 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
591 .guard_size = PINEVIEW_GUARD_WM,
592 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
593};
594
595static const struct intel_watermark_params pnv_cursor_wm = {
596 .fifo_size = PINEVIEW_CURSOR_FIFO,
597 .max_wm = PINEVIEW_CURSOR_MAX_WM,
598 .default_wm = PINEVIEW_CURSOR_DFT_WM,
599 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
600 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
601};
602
603static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
604 .fifo_size = PINEVIEW_CURSOR_FIFO,
605 .max_wm = PINEVIEW_CURSOR_MAX_WM,
606 .default_wm = PINEVIEW_CURSOR_DFT_WM,
607 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
608 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
609};
610
611static const struct intel_watermark_params i965_cursor_wm_info = {
612 .fifo_size = I965_CURSOR_FIFO,
613 .max_wm = I965_CURSOR_MAX_WM,
614 .default_wm = I965_CURSOR_DFT_WM,
615 .guard_size = 2,
616 .cacheline_size = I915_FIFO_LINE_SIZE,
617};
618
619static const struct intel_watermark_params i945_wm_info = {
620 .fifo_size = I945_FIFO_SIZE,
621 .max_wm = I915_MAX_WM,
622 .default_wm = 1,
623 .guard_size = 2,
624 .cacheline_size = I915_FIFO_LINE_SIZE,
625};
626
627static const struct intel_watermark_params i915_wm_info = {
628 .fifo_size = I915_FIFO_SIZE,
629 .max_wm = I915_MAX_WM,
630 .default_wm = 1,
631 .guard_size = 2,
632 .cacheline_size = I915_FIFO_LINE_SIZE,
633};
634
635static const struct intel_watermark_params i830_a_wm_info = {
636 .fifo_size = I855GM_FIFO_SIZE,
637 .max_wm = I915_MAX_WM,
638 .default_wm = 1,
639 .guard_size = 2,
640 .cacheline_size = I830_FIFO_LINE_SIZE,
641};
642
643static const struct intel_watermark_params i830_bc_wm_info = {
644 .fifo_size = I855GM_FIFO_SIZE,
645 .max_wm = I915_MAX_WM/2,
646 .default_wm = 1,
647 .guard_size = 2,
648 .cacheline_size = I830_FIFO_LINE_SIZE,
649};
650
651static const struct intel_watermark_params i845_wm_info = {
652 .fifo_size = I830_FIFO_SIZE,
653 .max_wm = I915_MAX_WM,
654 .default_wm = 1,
655 .guard_size = 2,
656 .cacheline_size = I830_FIFO_LINE_SIZE,
657};
658
659/**
660 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
661 * @pixel_rate: Pipe pixel rate in kHz
662 * @cpp: Plane bytes per pixel
663 * @latency: Memory wakeup latency in 0.1us units
664 *
665 * Compute the watermark using the method 1 or "small buffer"
666 * formula. The caller may additonally add extra cachelines
667 * to account for TLB misses and clock crossings.
668 *
669 * This method is concerned with the short term drain rate
670 * of the FIFO, ie. it does not account for blanking periods
671 * which would effectively reduce the average drain rate across
672 * a longer period. The name "small" refers to the fact the
673 * FIFO is relatively small compared to the amount of data
674 * fetched.
675 *
676 * The FIFO level vs. time graph might look something like:
677 *
678 * |\ |\
679 * | \ | \
680 * __---__---__ (- plane active, _ blanking)
681 * -> time
682 *
683 * or perhaps like this:
684 *
685 * |\|\ |\|\
686 * __----__----__ (- plane active, _ blanking)
687 * -> time
688 *
689 * Returns:
690 * The watermark in bytes
691 */
692static unsigned int intel_wm_method1(unsigned int pixel_rate,
693 unsigned int cpp,
694 unsigned int latency)
695{
696 u64 ret;
697
698 ret = mul_u32_u32(pixel_rate, cpp * latency);
699 ret = DIV_ROUND_UP_ULL(ret, 10000);
700
701 return ret;
702}
703
704/**
705 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
706 * @pixel_rate: Pipe pixel rate in kHz
707 * @htotal: Pipe horizontal total
708 * @width: Plane width in pixels
709 * @cpp: Plane bytes per pixel
710 * @latency: Memory wakeup latency in 0.1us units
711 *
712 * Compute the watermark using the method 2 or "large buffer"
713 * formula. The caller may additonally add extra cachelines
714 * to account for TLB misses and clock crossings.
715 *
716 * This method is concerned with the long term drain rate
717 * of the FIFO, ie. it does account for blanking periods
718 * which effectively reduce the average drain rate across
719 * a longer period. The name "large" refers to the fact the
720 * FIFO is relatively large compared to the amount of data
721 * fetched.
722 *
723 * The FIFO level vs. time graph might look something like:
724 *
725 * |\___ |\___
726 * | \___ | \___
727 * | \ | \
728 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
729 * -> time
730 *
731 * Returns:
732 * The watermark in bytes
733 */
734static unsigned int intel_wm_method2(unsigned int pixel_rate,
735 unsigned int htotal,
736 unsigned int width,
737 unsigned int cpp,
738 unsigned int latency)
739{
740 unsigned int ret;
741
742 /*
743 * FIXME remove once all users are computing
744 * watermarks in the correct place.
745 */
746 if (WARN_ON_ONCE(htotal == 0))
747 htotal = 1;
748
749 ret = (latency * pixel_rate) / (htotal * 10000);
750 ret = (ret + 1) * width * cpp;
751
752 return ret;
753}
754
755/**
756 * intel_calculate_wm - calculate watermark level
757 * @pixel_rate: pixel clock
758 * @wm: chip FIFO params
759 * @fifo_size: size of the FIFO buffer
760 * @cpp: bytes per pixel
761 * @latency_ns: memory latency for the platform
762 *
763 * Calculate the watermark level (the level at which the display plane will
764 * start fetching from memory again). Each chip has a different display
765 * FIFO size and allocation, so the caller needs to figure that out and pass
766 * in the correct intel_watermark_params structure.
767 *
768 * As the pixel clock runs, the FIFO will be drained at a rate that depends
769 * on the pixel size. When it reaches the watermark level, it'll start
770 * fetching FIFO line sized based chunks from memory until the FIFO fills
771 * past the watermark point. If the FIFO drains completely, a FIFO underrun
772 * will occur, and a display engine hang could result.
773 */
774static unsigned int intel_calculate_wm(int pixel_rate,
775 const struct intel_watermark_params *wm,
776 int fifo_size, int cpp,
777 unsigned int latency_ns)
778{
779 int entries, wm_size;
780
781 /*
782 * Note: we need to make sure we don't overflow for various clock &
783 * latency values.
784 * clocks go from a few thousand to several hundred thousand.
785 * latency is usually a few thousand
786 */
787 entries = intel_wm_method1(pixel_rate, cpp,
788 latency_ns / 100);
789 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
790 wm->guard_size;
791 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
792
793 wm_size = fifo_size - entries;
794 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
795
796 /* Don't promote wm_size to unsigned... */
797 if (wm_size > wm->max_wm)
798 wm_size = wm->max_wm;
799 if (wm_size <= 0)
800 wm_size = wm->default_wm;
801
802 /*
803 * Bspec seems to indicate that the value shouldn't be lower than
804 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
805 * Lets go for 8 which is the burst size since certain platforms
806 * already use a hardcoded 8 (which is what the spec says should be
807 * done).
808 */
809 if (wm_size <= 8)
810 wm_size = 8;
811
812 return wm_size;
813}
814
815static bool is_disabling(int old, int new, int threshold)
816{
817 return old >= threshold && new < threshold;
818}
819
820static bool is_enabling(int old, int new, int threshold)
821{
822 return old < threshold && new >= threshold;
823}
824
825static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
826{
827 return dev_priv->wm.max_level + 1;
828}
829
830static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
831 const struct intel_plane_state *plane_state)
832{
833 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
834
835 /* FIXME check the 'enable' instead */
836 if (!crtc_state->hw.active)
837 return false;
838
839 /*
840 * Treat cursor with fb as always visible since cursor updates
841 * can happen faster than the vrefresh rate, and the current
842 * watermark code doesn't handle that correctly. Cursor updates
843 * which set/clear the fb or change the cursor size are going
844 * to get throttled by intel_legacy_cursor_update() to work
845 * around this problem with the watermark code.
846 */
847 if (plane->id == PLANE_CURSOR)
848 return plane_state->hw.fb != NULL;
849 else
850 return plane_state->uapi.visible;
851}
852
853static bool intel_crtc_active(struct intel_crtc *crtc)
854{
855 /* Be paranoid as we can arrive here with only partial
856 * state retrieved from the hardware during setup.
857 *
858 * We can ditch the adjusted_mode.crtc_clock check as soon
859 * as Haswell has gained clock readout/fastboot support.
860 *
861 * We can ditch the crtc->primary->state->fb check as soon as we can
862 * properly reconstruct framebuffers.
863 *
864 * FIXME: The intel_crtc->active here should be switched to
865 * crtc->state->active once we have proper CRTC states wired up
866 * for atomic.
867 */
868 return crtc->active && crtc->base.primary->state->fb &&
869 crtc->config->hw.adjusted_mode.crtc_clock;
870}
871
872static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
873{
874 struct intel_crtc *crtc, *enabled = NULL;
875
876 for_each_intel_crtc(&dev_priv->drm, crtc) {
877 if (intel_crtc_active(crtc)) {
878 if (enabled)
879 return NULL;
880 enabled = crtc;
881 }
882 }
883
884 return enabled;
885}
886
887static void pnv_update_wm(struct intel_crtc *unused_crtc)
888{
889 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
890 struct intel_crtc *crtc;
891 const struct cxsr_latency *latency;
892 u32 reg;
893 unsigned int wm;
894
895 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
896 dev_priv->is_ddr3,
897 dev_priv->fsb_freq,
898 dev_priv->mem_freq);
899 if (!latency) {
900 drm_dbg_kms(&dev_priv->drm,
901 "Unknown FSB/MEM found, disable CxSR\n");
902 intel_set_memory_cxsr(dev_priv, false);
903 return;
904 }
905
906 crtc = single_enabled_crtc(dev_priv);
907 if (crtc) {
908 const struct drm_display_mode *adjusted_mode =
909 &crtc->config->hw.adjusted_mode;
910 const struct drm_framebuffer *fb =
911 crtc->base.primary->state->fb;
912 int cpp = fb->format->cpp[0];
913 int clock = adjusted_mode->crtc_clock;
914
915 /* Display SR */
916 wm = intel_calculate_wm(clock, &pnv_display_wm,
917 pnv_display_wm.fifo_size,
918 cpp, latency->display_sr);
919 reg = I915_READ(DSPFW1);
920 reg &= ~DSPFW_SR_MASK;
921 reg |= FW_WM(wm, SR);
922 I915_WRITE(DSPFW1, reg);
923 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
924
925 /* cursor SR */
926 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
927 pnv_display_wm.fifo_size,
928 4, latency->cursor_sr);
929 reg = I915_READ(DSPFW3);
930 reg &= ~DSPFW_CURSOR_SR_MASK;
931 reg |= FW_WM(wm, CURSOR_SR);
932 I915_WRITE(DSPFW3, reg);
933
934 /* Display HPLL off SR */
935 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
936 pnv_display_hplloff_wm.fifo_size,
937 cpp, latency->display_hpll_disable);
938 reg = I915_READ(DSPFW3);
939 reg &= ~DSPFW_HPLL_SR_MASK;
940 reg |= FW_WM(wm, HPLL_SR);
941 I915_WRITE(DSPFW3, reg);
942
943 /* cursor HPLL off SR */
944 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 4, latency->cursor_hpll_disable);
947 reg = I915_READ(DSPFW3);
948 reg &= ~DSPFW_HPLL_CURSOR_MASK;
949 reg |= FW_WM(wm, HPLL_CURSOR);
950 I915_WRITE(DSPFW3, reg);
951 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
952
953 intel_set_memory_cxsr(dev_priv, true);
954 } else {
955 intel_set_memory_cxsr(dev_priv, false);
956 }
957}
958
959/*
960 * Documentation says:
961 * "If the line size is small, the TLB fetches can get in the way of the
962 * data fetches, causing some lag in the pixel data return which is not
963 * accounted for in the above formulas. The following adjustment only
964 * needs to be applied if eight whole lines fit in the buffer at once.
965 * The WM is adjusted upwards by the difference between the FIFO size
966 * and the size of 8 whole lines. This adjustment is always performed
967 * in the actual pixel depth regardless of whether FBC is enabled or not."
968 */
969static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
970{
971 int tlb_miss = fifo_size * 64 - width * cpp * 8;
972
973 return max(0, tlb_miss);
974}
975
976static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
977 const struct g4x_wm_values *wm)
978{
979 enum pipe pipe;
980
981 for_each_pipe(dev_priv, pipe)
982 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
983
984 I915_WRITE(DSPFW1,
985 FW_WM(wm->sr.plane, SR) |
986 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
987 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
988 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
989 I915_WRITE(DSPFW2,
990 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
991 FW_WM(wm->sr.fbc, FBC_SR) |
992 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
993 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
994 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
995 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
996 I915_WRITE(DSPFW3,
997 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
998 FW_WM(wm->sr.cursor, CURSOR_SR) |
999 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1000 FW_WM(wm->hpll.plane, HPLL_SR));
1001
1002 POSTING_READ(DSPFW1);
1003}
1004
1005#define FW_WM_VLV(value, plane) \
1006 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1007
1008static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1009 const struct vlv_wm_values *wm)
1010{
1011 enum pipe pipe;
1012
1013 for_each_pipe(dev_priv, pipe) {
1014 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1015
1016 I915_WRITE(VLV_DDL(pipe),
1017 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1018 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1019 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1020 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1021 }
1022
1023 /*
1024 * Zero the (unused) WM1 watermarks, and also clear all the
1025 * high order bits so that there are no out of bounds values
1026 * present in the registers during the reprogramming.
1027 */
1028 I915_WRITE(DSPHOWM, 0);
1029 I915_WRITE(DSPHOWM1, 0);
1030 I915_WRITE(DSPFW4, 0);
1031 I915_WRITE(DSPFW5, 0);
1032 I915_WRITE(DSPFW6, 0);
1033
1034 I915_WRITE(DSPFW1,
1035 FW_WM(wm->sr.plane, SR) |
1036 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1037 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1038 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1039 I915_WRITE(DSPFW2,
1040 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1041 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1042 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1043 I915_WRITE(DSPFW3,
1044 FW_WM(wm->sr.cursor, CURSOR_SR));
1045
1046 if (IS_CHERRYVIEW(dev_priv)) {
1047 I915_WRITE(DSPFW7_CHV,
1048 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1049 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1050 I915_WRITE(DSPFW8_CHV,
1051 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1052 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1053 I915_WRITE(DSPFW9_CHV,
1054 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1055 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1056 I915_WRITE(DSPHOWM,
1057 FW_WM(wm->sr.plane >> 9, SR_HI) |
1058 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1059 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1060 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1061 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1062 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1063 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1064 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1065 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1066 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1067 } else {
1068 I915_WRITE(DSPFW7,
1069 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1070 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1071 I915_WRITE(DSPHOWM,
1072 FW_WM(wm->sr.plane >> 9, SR_HI) |
1073 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1074 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1075 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1076 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1077 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1078 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1079 }
1080
1081 POSTING_READ(DSPFW1);
1082}
1083
1084#undef FW_WM_VLV
1085
1086static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1087{
1088 /* all latencies in usec */
1089 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1090 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1091 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1092
1093 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1094}
1095
1096static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1097{
1098 /*
1099 * DSPCNTR[13] supposedly controls whether the
1100 * primary plane can use the FIFO space otherwise
1101 * reserved for the sprite plane. It's not 100% clear
1102 * what the actual FIFO size is, but it looks like we
1103 * can happily set both primary and sprite watermarks
1104 * up to 127 cachelines. So that would seem to mean
1105 * that either DSPCNTR[13] doesn't do anything, or that
1106 * the total FIFO is >= 256 cachelines in size. Either
1107 * way, we don't seem to have to worry about this
1108 * repartitioning as the maximum watermark value the
1109 * register can hold for each plane is lower than the
1110 * minimum FIFO size.
1111 */
1112 switch (plane_id) {
1113 case PLANE_CURSOR:
1114 return 63;
1115 case PLANE_PRIMARY:
1116 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1117 case PLANE_SPRITE0:
1118 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1119 default:
1120 MISSING_CASE(plane_id);
1121 return 0;
1122 }
1123}
1124
1125static int g4x_fbc_fifo_size(int level)
1126{
1127 switch (level) {
1128 case G4X_WM_LEVEL_SR:
1129 return 7;
1130 case G4X_WM_LEVEL_HPLL:
1131 return 15;
1132 default:
1133 MISSING_CASE(level);
1134 return 0;
1135 }
1136}
1137
1138static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1139 const struct intel_plane_state *plane_state,
1140 int level)
1141{
1142 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1143 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1144 const struct drm_display_mode *adjusted_mode =
1145 &crtc_state->hw.adjusted_mode;
1146 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1147 unsigned int clock, htotal, cpp, width, wm;
1148
1149 if (latency == 0)
1150 return USHRT_MAX;
1151
1152 if (!intel_wm_plane_visible(crtc_state, plane_state))
1153 return 0;
1154
1155 cpp = plane_state->hw.fb->format->cpp[0];
1156
1157 /*
1158 * Not 100% sure which way ELK should go here as the
1159 * spec only says CL/CTG should assume 32bpp and BW
1160 * doesn't need to. But as these things followed the
1161 * mobile vs. desktop lines on gen3 as well, let's
1162 * assume ELK doesn't need this.
1163 *
1164 * The spec also fails to list such a restriction for
1165 * the HPLL watermark, which seems a little strange.
1166 * Let's use 32bpp for the HPLL watermark as well.
1167 */
1168 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1169 level != G4X_WM_LEVEL_NORMAL)
1170 cpp = max(cpp, 4u);
1171
1172 clock = adjusted_mode->crtc_clock;
1173 htotal = adjusted_mode->crtc_htotal;
1174
1175 width = drm_rect_width(&plane_state->uapi.dst);
1176
1177 if (plane->id == PLANE_CURSOR) {
1178 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1179 } else if (plane->id == PLANE_PRIMARY &&
1180 level == G4X_WM_LEVEL_NORMAL) {
1181 wm = intel_wm_method1(clock, cpp, latency);
1182 } else {
1183 unsigned int small, large;
1184
1185 small = intel_wm_method1(clock, cpp, latency);
1186 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1187
1188 wm = min(small, large);
1189 }
1190
1191 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1192 width, cpp);
1193
1194 wm = DIV_ROUND_UP(wm, 64) + 2;
1195
1196 return min_t(unsigned int, wm, USHRT_MAX);
1197}
1198
1199static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1200 int level, enum plane_id plane_id, u16 value)
1201{
1202 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1203 bool dirty = false;
1204
1205 for (; level < intel_wm_num_levels(dev_priv); level++) {
1206 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1207
1208 dirty |= raw->plane[plane_id] != value;
1209 raw->plane[plane_id] = value;
1210 }
1211
1212 return dirty;
1213}
1214
1215static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1216 int level, u16 value)
1217{
1218 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1219 bool dirty = false;
1220
1221 /* NORMAL level doesn't have an FBC watermark */
1222 level = max(level, G4X_WM_LEVEL_SR);
1223
1224 for (; level < intel_wm_num_levels(dev_priv); level++) {
1225 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1226
1227 dirty |= raw->fbc != value;
1228 raw->fbc = value;
1229 }
1230
1231 return dirty;
1232}
1233
1234static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1235 const struct intel_plane_state *plane_state,
1236 u32 pri_val);
1237
1238static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1239 const struct intel_plane_state *plane_state)
1240{
1241 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1242 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1243 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1244 enum plane_id plane_id = plane->id;
1245 bool dirty = false;
1246 int level;
1247
1248 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1249 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1250 if (plane_id == PLANE_PRIMARY)
1251 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1252 goto out;
1253 }
1254
1255 for (level = 0; level < num_levels; level++) {
1256 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1257 int wm, max_wm;
1258
1259 wm = g4x_compute_wm(crtc_state, plane_state, level);
1260 max_wm = g4x_plane_fifo_size(plane_id, level);
1261
1262 if (wm > max_wm)
1263 break;
1264
1265 dirty |= raw->plane[plane_id] != wm;
1266 raw->plane[plane_id] = wm;
1267
1268 if (plane_id != PLANE_PRIMARY ||
1269 level == G4X_WM_LEVEL_NORMAL)
1270 continue;
1271
1272 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1273 raw->plane[plane_id]);
1274 max_wm = g4x_fbc_fifo_size(level);
1275
1276 /*
1277 * FBC wm is not mandatory as we
1278 * can always just disable its use.
1279 */
1280 if (wm > max_wm)
1281 wm = USHRT_MAX;
1282
1283 dirty |= raw->fbc != wm;
1284 raw->fbc = wm;
1285 }
1286
1287 /* mark watermarks as invalid */
1288 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1289
1290 if (plane_id == PLANE_PRIMARY)
1291 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1292
1293 out:
1294 if (dirty) {
1295 drm_dbg_kms(&dev_priv->drm,
1296 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1297 plane->base.name,
1298 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1299 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1300 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1301
1302 if (plane_id == PLANE_PRIMARY)
1303 drm_dbg_kms(&dev_priv->drm,
1304 "FBC watermarks: SR=%d, HPLL=%d\n",
1305 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1306 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1307 }
1308
1309 return dirty;
1310}
1311
1312static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1313 enum plane_id plane_id, int level)
1314{
1315 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1316
1317 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1318}
1319
1320static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1321 int level)
1322{
1323 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1324
1325 if (level > dev_priv->wm.max_level)
1326 return false;
1327
1328 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1329 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1330 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1331}
1332
1333/* mark all levels starting from 'level' as invalid */
1334static void g4x_invalidate_wms(struct intel_crtc *crtc,
1335 struct g4x_wm_state *wm_state, int level)
1336{
1337 if (level <= G4X_WM_LEVEL_NORMAL) {
1338 enum plane_id plane_id;
1339
1340 for_each_plane_id_on_crtc(crtc, plane_id)
1341 wm_state->wm.plane[plane_id] = USHRT_MAX;
1342 }
1343
1344 if (level <= G4X_WM_LEVEL_SR) {
1345 wm_state->cxsr = false;
1346 wm_state->sr.cursor = USHRT_MAX;
1347 wm_state->sr.plane = USHRT_MAX;
1348 wm_state->sr.fbc = USHRT_MAX;
1349 }
1350
1351 if (level <= G4X_WM_LEVEL_HPLL) {
1352 wm_state->hpll_en = false;
1353 wm_state->hpll.cursor = USHRT_MAX;
1354 wm_state->hpll.plane = USHRT_MAX;
1355 wm_state->hpll.fbc = USHRT_MAX;
1356 }
1357}
1358
1359static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1360 int level)
1361{
1362 if (level < G4X_WM_LEVEL_SR)
1363 return false;
1364
1365 if (level >= G4X_WM_LEVEL_SR &&
1366 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1367 return false;
1368
1369 if (level >= G4X_WM_LEVEL_HPLL &&
1370 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1371 return false;
1372
1373 return true;
1374}
1375
1376static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1377{
1378 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1379 struct intel_atomic_state *state =
1380 to_intel_atomic_state(crtc_state->uapi.state);
1381 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1382 int num_active_planes = hweight8(crtc_state->active_planes &
1383 ~BIT(PLANE_CURSOR));
1384 const struct g4x_pipe_wm *raw;
1385 const struct intel_plane_state *old_plane_state;
1386 const struct intel_plane_state *new_plane_state;
1387 struct intel_plane *plane;
1388 enum plane_id plane_id;
1389 int i, level;
1390 unsigned int dirty = 0;
1391
1392 for_each_oldnew_intel_plane_in_state(state, plane,
1393 old_plane_state,
1394 new_plane_state, i) {
1395 if (new_plane_state->hw.crtc != &crtc->base &&
1396 old_plane_state->hw.crtc != &crtc->base)
1397 continue;
1398
1399 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1400 dirty |= BIT(plane->id);
1401 }
1402
1403 if (!dirty)
1404 return 0;
1405
1406 level = G4X_WM_LEVEL_NORMAL;
1407 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1408 goto out;
1409
1410 raw = &crtc_state->wm.g4x.raw[level];
1411 for_each_plane_id_on_crtc(crtc, plane_id)
1412 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1413
1414 level = G4X_WM_LEVEL_SR;
1415 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1416 goto out;
1417
1418 raw = &crtc_state->wm.g4x.raw[level];
1419 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1420 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1421 wm_state->sr.fbc = raw->fbc;
1422
1423 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1424
1425 level = G4X_WM_LEVEL_HPLL;
1426 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1427 goto out;
1428
1429 raw = &crtc_state->wm.g4x.raw[level];
1430 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1431 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1432 wm_state->hpll.fbc = raw->fbc;
1433
1434 wm_state->hpll_en = wm_state->cxsr;
1435
1436 level++;
1437
1438 out:
1439 if (level == G4X_WM_LEVEL_NORMAL)
1440 return -EINVAL;
1441
1442 /* invalidate the higher levels */
1443 g4x_invalidate_wms(crtc, wm_state, level);
1444
1445 /*
1446 * Determine if the FBC watermark(s) can be used. IF
1447 * this isn't the case we prefer to disable the FBC
1448 * watermark(s) rather than disable the SR/HPLL
1449 * level(s) entirely. 'level-1' is the highest valid
1450 * level here.
1451 */
1452 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1453
1454 return 0;
1455}
1456
1457static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1458{
1459 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1460 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1461 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1462 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1463 struct intel_atomic_state *intel_state =
1464 to_intel_atomic_state(new_crtc_state->uapi.state);
1465 const struct intel_crtc_state *old_crtc_state =
1466 intel_atomic_get_old_crtc_state(intel_state, crtc);
1467 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1468 enum plane_id plane_id;
1469
1470 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1471 *intermediate = *optimal;
1472
1473 intermediate->cxsr = false;
1474 intermediate->hpll_en = false;
1475 goto out;
1476 }
1477
1478 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1479 !new_crtc_state->disable_cxsr;
1480 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1481 !new_crtc_state->disable_cxsr;
1482 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1483
1484 for_each_plane_id_on_crtc(crtc, plane_id) {
1485 intermediate->wm.plane[plane_id] =
1486 max(optimal->wm.plane[plane_id],
1487 active->wm.plane[plane_id]);
1488
1489 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1490 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1491 }
1492
1493 intermediate->sr.plane = max(optimal->sr.plane,
1494 active->sr.plane);
1495 intermediate->sr.cursor = max(optimal->sr.cursor,
1496 active->sr.cursor);
1497 intermediate->sr.fbc = max(optimal->sr.fbc,
1498 active->sr.fbc);
1499
1500 intermediate->hpll.plane = max(optimal->hpll.plane,
1501 active->hpll.plane);
1502 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1503 active->hpll.cursor);
1504 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1505 active->hpll.fbc);
1506
1507 drm_WARN_ON(&dev_priv->drm,
1508 (intermediate->sr.plane >
1509 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1510 intermediate->sr.cursor >
1511 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1512 intermediate->cxsr);
1513 drm_WARN_ON(&dev_priv->drm,
1514 (intermediate->sr.plane >
1515 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1516 intermediate->sr.cursor >
1517 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1518 intermediate->hpll_en);
1519
1520 drm_WARN_ON(&dev_priv->drm,
1521 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1522 intermediate->fbc_en && intermediate->cxsr);
1523 drm_WARN_ON(&dev_priv->drm,
1524 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1525 intermediate->fbc_en && intermediate->hpll_en);
1526
1527out:
1528 /*
1529 * If our intermediate WM are identical to the final WM, then we can
1530 * omit the post-vblank programming; only update if it's different.
1531 */
1532 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1533 new_crtc_state->wm.need_postvbl_update = true;
1534
1535 return 0;
1536}
1537
1538static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1539 struct g4x_wm_values *wm)
1540{
1541 struct intel_crtc *crtc;
1542 int num_active_pipes = 0;
1543
1544 wm->cxsr = true;
1545 wm->hpll_en = true;
1546 wm->fbc_en = true;
1547
1548 for_each_intel_crtc(&dev_priv->drm, crtc) {
1549 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1550
1551 if (!crtc->active)
1552 continue;
1553
1554 if (!wm_state->cxsr)
1555 wm->cxsr = false;
1556 if (!wm_state->hpll_en)
1557 wm->hpll_en = false;
1558 if (!wm_state->fbc_en)
1559 wm->fbc_en = false;
1560
1561 num_active_pipes++;
1562 }
1563
1564 if (num_active_pipes != 1) {
1565 wm->cxsr = false;
1566 wm->hpll_en = false;
1567 wm->fbc_en = false;
1568 }
1569
1570 for_each_intel_crtc(&dev_priv->drm, crtc) {
1571 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1572 enum pipe pipe = crtc->pipe;
1573
1574 wm->pipe[pipe] = wm_state->wm;
1575 if (crtc->active && wm->cxsr)
1576 wm->sr = wm_state->sr;
1577 if (crtc->active && wm->hpll_en)
1578 wm->hpll = wm_state->hpll;
1579 }
1580}
1581
1582static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1583{
1584 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1585 struct g4x_wm_values new_wm = {};
1586
1587 g4x_merge_wm(dev_priv, &new_wm);
1588
1589 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1590 return;
1591
1592 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1593 _intel_set_memory_cxsr(dev_priv, false);
1594
1595 g4x_write_wm_values(dev_priv, &new_wm);
1596
1597 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1598 _intel_set_memory_cxsr(dev_priv, true);
1599
1600 *old_wm = new_wm;
1601}
1602
1603static void g4x_initial_watermarks(struct intel_atomic_state *state,
1604 struct intel_crtc *crtc)
1605{
1606 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1607 const struct intel_crtc_state *crtc_state =
1608 intel_atomic_get_new_crtc_state(state, crtc);
1609
1610 mutex_lock(&dev_priv->wm.wm_mutex);
1611 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1612 g4x_program_watermarks(dev_priv);
1613 mutex_unlock(&dev_priv->wm.wm_mutex);
1614}
1615
1616static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1617 struct intel_crtc *crtc)
1618{
1619 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1620 const struct intel_crtc_state *crtc_state =
1621 intel_atomic_get_new_crtc_state(state, crtc);
1622
1623 if (!crtc_state->wm.need_postvbl_update)
1624 return;
1625
1626 mutex_lock(&dev_priv->wm.wm_mutex);
1627 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1628 g4x_program_watermarks(dev_priv);
1629 mutex_unlock(&dev_priv->wm.wm_mutex);
1630}
1631
1632/* latency must be in 0.1us units. */
1633static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1634 unsigned int htotal,
1635 unsigned int width,
1636 unsigned int cpp,
1637 unsigned int latency)
1638{
1639 unsigned int ret;
1640
1641 ret = intel_wm_method2(pixel_rate, htotal,
1642 width, cpp, latency);
1643 ret = DIV_ROUND_UP(ret, 64);
1644
1645 return ret;
1646}
1647
1648static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1649{
1650 /* all latencies in usec */
1651 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1652
1653 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1654
1655 if (IS_CHERRYVIEW(dev_priv)) {
1656 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1657 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1658
1659 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1660 }
1661}
1662
1663static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1664 const struct intel_plane_state *plane_state,
1665 int level)
1666{
1667 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1668 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1669 const struct drm_display_mode *adjusted_mode =
1670 &crtc_state->hw.adjusted_mode;
1671 unsigned int clock, htotal, cpp, width, wm;
1672
1673 if (dev_priv->wm.pri_latency[level] == 0)
1674 return USHRT_MAX;
1675
1676 if (!intel_wm_plane_visible(crtc_state, plane_state))
1677 return 0;
1678
1679 cpp = plane_state->hw.fb->format->cpp[0];
1680 clock = adjusted_mode->crtc_clock;
1681 htotal = adjusted_mode->crtc_htotal;
1682 width = crtc_state->pipe_src_w;
1683
1684 if (plane->id == PLANE_CURSOR) {
1685 /*
1686 * FIXME the formula gives values that are
1687 * too big for the cursor FIFO, and hence we
1688 * would never be able to use cursors. For
1689 * now just hardcode the watermark.
1690 */
1691 wm = 63;
1692 } else {
1693 wm = vlv_wm_method2(clock, htotal, width, cpp,
1694 dev_priv->wm.pri_latency[level] * 10);
1695 }
1696
1697 return min_t(unsigned int, wm, USHRT_MAX);
1698}
1699
1700static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1701{
1702 return (active_planes & (BIT(PLANE_SPRITE0) |
1703 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1704}
1705
1706static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1707{
1708 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1709 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1710 const struct g4x_pipe_wm *raw =
1711 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1712 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1713 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1714 int num_active_planes = hweight8(active_planes);
1715 const int fifo_size = 511;
1716 int fifo_extra, fifo_left = fifo_size;
1717 int sprite0_fifo_extra = 0;
1718 unsigned int total_rate;
1719 enum plane_id plane_id;
1720
1721 /*
1722 * When enabling sprite0 after sprite1 has already been enabled
1723 * we tend to get an underrun unless sprite0 already has some
1724 * FIFO space allcoated. Hence we always allocate at least one
1725 * cacheline for sprite0 whenever sprite1 is enabled.
1726 *
1727 * All other plane enable sequences appear immune to this problem.
1728 */
1729 if (vlv_need_sprite0_fifo_workaround(active_planes))
1730 sprite0_fifo_extra = 1;
1731
1732 total_rate = raw->plane[PLANE_PRIMARY] +
1733 raw->plane[PLANE_SPRITE0] +
1734 raw->plane[PLANE_SPRITE1] +
1735 sprite0_fifo_extra;
1736
1737 if (total_rate > fifo_size)
1738 return -EINVAL;
1739
1740 if (total_rate == 0)
1741 total_rate = 1;
1742
1743 for_each_plane_id_on_crtc(crtc, plane_id) {
1744 unsigned int rate;
1745
1746 if ((active_planes & BIT(plane_id)) == 0) {
1747 fifo_state->plane[plane_id] = 0;
1748 continue;
1749 }
1750
1751 rate = raw->plane[plane_id];
1752 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1753 fifo_left -= fifo_state->plane[plane_id];
1754 }
1755
1756 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1757 fifo_left -= sprite0_fifo_extra;
1758
1759 fifo_state->plane[PLANE_CURSOR] = 63;
1760
1761 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1762
1763 /* spread the remainder evenly */
1764 for_each_plane_id_on_crtc(crtc, plane_id) {
1765 int plane_extra;
1766
1767 if (fifo_left == 0)
1768 break;
1769
1770 if ((active_planes & BIT(plane_id)) == 0)
1771 continue;
1772
1773 plane_extra = min(fifo_extra, fifo_left);
1774 fifo_state->plane[plane_id] += plane_extra;
1775 fifo_left -= plane_extra;
1776 }
1777
1778 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1779
1780 /* give it all to the first plane if none are active */
1781 if (active_planes == 0) {
1782 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1783 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1784 }
1785
1786 return 0;
1787}
1788
1789/* mark all levels starting from 'level' as invalid */
1790static void vlv_invalidate_wms(struct intel_crtc *crtc,
1791 struct vlv_wm_state *wm_state, int level)
1792{
1793 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1794
1795 for (; level < intel_wm_num_levels(dev_priv); level++) {
1796 enum plane_id plane_id;
1797
1798 for_each_plane_id_on_crtc(crtc, plane_id)
1799 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1800
1801 wm_state->sr[level].cursor = USHRT_MAX;
1802 wm_state->sr[level].plane = USHRT_MAX;
1803 }
1804}
1805
1806static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1807{
1808 if (wm > fifo_size)
1809 return USHRT_MAX;
1810 else
1811 return fifo_size - wm;
1812}
1813
1814/*
1815 * Starting from 'level' set all higher
1816 * levels to 'value' in the "raw" watermarks.
1817 */
1818static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1819 int level, enum plane_id plane_id, u16 value)
1820{
1821 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1822 int num_levels = intel_wm_num_levels(dev_priv);
1823 bool dirty = false;
1824
1825 for (; level < num_levels; level++) {
1826 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1827
1828 dirty |= raw->plane[plane_id] != value;
1829 raw->plane[plane_id] = value;
1830 }
1831
1832 return dirty;
1833}
1834
1835static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1836 const struct intel_plane_state *plane_state)
1837{
1838 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1839 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1840 enum plane_id plane_id = plane->id;
1841 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1842 int level;
1843 bool dirty = false;
1844
1845 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1846 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1847 goto out;
1848 }
1849
1850 for (level = 0; level < num_levels; level++) {
1851 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1852 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1853 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1854
1855 if (wm > max_wm)
1856 break;
1857
1858 dirty |= raw->plane[plane_id] != wm;
1859 raw->plane[plane_id] = wm;
1860 }
1861
1862 /* mark all higher levels as invalid */
1863 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1864
1865out:
1866 if (dirty)
1867 drm_dbg_kms(&dev_priv->drm,
1868 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1869 plane->base.name,
1870 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1871 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1872 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1873
1874 return dirty;
1875}
1876
1877static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1878 enum plane_id plane_id, int level)
1879{
1880 const struct g4x_pipe_wm *raw =
1881 &crtc_state->wm.vlv.raw[level];
1882 const struct vlv_fifo_state *fifo_state =
1883 &crtc_state->wm.vlv.fifo_state;
1884
1885 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1886}
1887
1888static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1889{
1890 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1891 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1892 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1893 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1894}
1895
1896static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1897{
1898 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1899 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1900 struct intel_atomic_state *state =
1901 to_intel_atomic_state(crtc_state->uapi.state);
1902 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1903 const struct vlv_fifo_state *fifo_state =
1904 &crtc_state->wm.vlv.fifo_state;
1905 int num_active_planes = hweight8(crtc_state->active_planes &
1906 ~BIT(PLANE_CURSOR));
1907 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1908 const struct intel_plane_state *old_plane_state;
1909 const struct intel_plane_state *new_plane_state;
1910 struct intel_plane *plane;
1911 enum plane_id plane_id;
1912 int level, ret, i;
1913 unsigned int dirty = 0;
1914
1915 for_each_oldnew_intel_plane_in_state(state, plane,
1916 old_plane_state,
1917 new_plane_state, i) {
1918 if (new_plane_state->hw.crtc != &crtc->base &&
1919 old_plane_state->hw.crtc != &crtc->base)
1920 continue;
1921
1922 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1923 dirty |= BIT(plane->id);
1924 }
1925
1926 /*
1927 * DSPARB registers may have been reset due to the
1928 * power well being turned off. Make sure we restore
1929 * them to a consistent state even if no primary/sprite
1930 * planes are initially active.
1931 */
1932 if (needs_modeset)
1933 crtc_state->fifo_changed = true;
1934
1935 if (!dirty)
1936 return 0;
1937
1938 /* cursor changes don't warrant a FIFO recompute */
1939 if (dirty & ~BIT(PLANE_CURSOR)) {
1940 const struct intel_crtc_state *old_crtc_state =
1941 intel_atomic_get_old_crtc_state(state, crtc);
1942 const struct vlv_fifo_state *old_fifo_state =
1943 &old_crtc_state->wm.vlv.fifo_state;
1944
1945 ret = vlv_compute_fifo(crtc_state);
1946 if (ret)
1947 return ret;
1948
1949 if (needs_modeset ||
1950 memcmp(old_fifo_state, fifo_state,
1951 sizeof(*fifo_state)) != 0)
1952 crtc_state->fifo_changed = true;
1953 }
1954
1955 /* initially allow all levels */
1956 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1957 /*
1958 * Note that enabling cxsr with no primary/sprite planes
1959 * enabled can wedge the pipe. Hence we only allow cxsr
1960 * with exactly one enabled primary/sprite plane.
1961 */
1962 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1963
1964 for (level = 0; level < wm_state->num_levels; level++) {
1965 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1966 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1967
1968 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1969 break;
1970
1971 for_each_plane_id_on_crtc(crtc, plane_id) {
1972 wm_state->wm[level].plane[plane_id] =
1973 vlv_invert_wm_value(raw->plane[plane_id],
1974 fifo_state->plane[plane_id]);
1975 }
1976
1977 wm_state->sr[level].plane =
1978 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1979 raw->plane[PLANE_SPRITE0],
1980 raw->plane[PLANE_SPRITE1]),
1981 sr_fifo_size);
1982
1983 wm_state->sr[level].cursor =
1984 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1985 63);
1986 }
1987
1988 if (level == 0)
1989 return -EINVAL;
1990
1991 /* limit to only levels we can actually handle */
1992 wm_state->num_levels = level;
1993
1994 /* invalidate the higher levels */
1995 vlv_invalidate_wms(crtc, wm_state, level);
1996
1997 return 0;
1998}
1999
2000#define VLV_FIFO(plane, value) \
2001 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2002
2003static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2004 struct intel_crtc *crtc)
2005{
2006 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2007 struct intel_uncore *uncore = &dev_priv->uncore;
2008 const struct intel_crtc_state *crtc_state =
2009 intel_atomic_get_new_crtc_state(state, crtc);
2010 const struct vlv_fifo_state *fifo_state =
2011 &crtc_state->wm.vlv.fifo_state;
2012 int sprite0_start, sprite1_start, fifo_size;
2013 u32 dsparb, dsparb2, dsparb3;
2014
2015 if (!crtc_state->fifo_changed)
2016 return;
2017
2018 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2019 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2020 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2021
2022 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2023 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2024
2025 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2026
2027 /*
2028 * uncore.lock serves a double purpose here. It allows us to
2029 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2030 * it protects the DSPARB registers from getting clobbered by
2031 * parallel updates from multiple pipes.
2032 *
2033 * intel_pipe_update_start() has already disabled interrupts
2034 * for us, so a plain spin_lock() is sufficient here.
2035 */
2036 spin_lock(&uncore->lock);
2037
2038 switch (crtc->pipe) {
2039 case PIPE_A:
2040 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2041 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2042
2043 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2044 VLV_FIFO(SPRITEB, 0xff));
2045 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2046 VLV_FIFO(SPRITEB, sprite1_start));
2047
2048 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2049 VLV_FIFO(SPRITEB_HI, 0x1));
2050 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2051 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2052
2053 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2054 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2055 break;
2056 case PIPE_B:
2057 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2058 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2059
2060 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2061 VLV_FIFO(SPRITED, 0xff));
2062 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2063 VLV_FIFO(SPRITED, sprite1_start));
2064
2065 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2066 VLV_FIFO(SPRITED_HI, 0xff));
2067 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2068 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2069
2070 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2071 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2072 break;
2073 case PIPE_C:
2074 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2075 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2076
2077 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2078 VLV_FIFO(SPRITEF, 0xff));
2079 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2080 VLV_FIFO(SPRITEF, sprite1_start));
2081
2082 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2083 VLV_FIFO(SPRITEF_HI, 0xff));
2084 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2085 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2086
2087 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2088 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2089 break;
2090 default:
2091 break;
2092 }
2093
2094 intel_uncore_posting_read_fw(uncore, DSPARB);
2095
2096 spin_unlock(&uncore->lock);
2097}
2098
2099#undef VLV_FIFO
2100
2101static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2102{
2103 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2104 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2105 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2106 struct intel_atomic_state *intel_state =
2107 to_intel_atomic_state(new_crtc_state->uapi.state);
2108 const struct intel_crtc_state *old_crtc_state =
2109 intel_atomic_get_old_crtc_state(intel_state, crtc);
2110 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2111 int level;
2112
2113 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2114 *intermediate = *optimal;
2115
2116 intermediate->cxsr = false;
2117 goto out;
2118 }
2119
2120 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2121 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2122 !new_crtc_state->disable_cxsr;
2123
2124 for (level = 0; level < intermediate->num_levels; level++) {
2125 enum plane_id plane_id;
2126
2127 for_each_plane_id_on_crtc(crtc, plane_id) {
2128 intermediate->wm[level].plane[plane_id] =
2129 min(optimal->wm[level].plane[plane_id],
2130 active->wm[level].plane[plane_id]);
2131 }
2132
2133 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2134 active->sr[level].plane);
2135 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2136 active->sr[level].cursor);
2137 }
2138
2139 vlv_invalidate_wms(crtc, intermediate, level);
2140
2141out:
2142 /*
2143 * If our intermediate WM are identical to the final WM, then we can
2144 * omit the post-vblank programming; only update if it's different.
2145 */
2146 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2147 new_crtc_state->wm.need_postvbl_update = true;
2148
2149 return 0;
2150}
2151
2152static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2153 struct vlv_wm_values *wm)
2154{
2155 struct intel_crtc *crtc;
2156 int num_active_pipes = 0;
2157
2158 wm->level = dev_priv->wm.max_level;
2159 wm->cxsr = true;
2160
2161 for_each_intel_crtc(&dev_priv->drm, crtc) {
2162 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2163
2164 if (!crtc->active)
2165 continue;
2166
2167 if (!wm_state->cxsr)
2168 wm->cxsr = false;
2169
2170 num_active_pipes++;
2171 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2172 }
2173
2174 if (num_active_pipes != 1)
2175 wm->cxsr = false;
2176
2177 if (num_active_pipes > 1)
2178 wm->level = VLV_WM_LEVEL_PM2;
2179
2180 for_each_intel_crtc(&dev_priv->drm, crtc) {
2181 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2182 enum pipe pipe = crtc->pipe;
2183
2184 wm->pipe[pipe] = wm_state->wm[wm->level];
2185 if (crtc->active && wm->cxsr)
2186 wm->sr = wm_state->sr[wm->level];
2187
2188 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2189 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2190 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2191 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2192 }
2193}
2194
2195static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2196{
2197 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2198 struct vlv_wm_values new_wm = {};
2199
2200 vlv_merge_wm(dev_priv, &new_wm);
2201
2202 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2203 return;
2204
2205 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2206 chv_set_memory_dvfs(dev_priv, false);
2207
2208 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209 chv_set_memory_pm5(dev_priv, false);
2210
2211 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2212 _intel_set_memory_cxsr(dev_priv, false);
2213
2214 vlv_write_wm_values(dev_priv, &new_wm);
2215
2216 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2217 _intel_set_memory_cxsr(dev_priv, true);
2218
2219 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220 chv_set_memory_pm5(dev_priv, true);
2221
2222 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2223 chv_set_memory_dvfs(dev_priv, true);
2224
2225 *old_wm = new_wm;
2226}
2227
2228static void vlv_initial_watermarks(struct intel_atomic_state *state,
2229 struct intel_crtc *crtc)
2230{
2231 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2232 const struct intel_crtc_state *crtc_state =
2233 intel_atomic_get_new_crtc_state(state, crtc);
2234
2235 mutex_lock(&dev_priv->wm.wm_mutex);
2236 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2237 vlv_program_watermarks(dev_priv);
2238 mutex_unlock(&dev_priv->wm.wm_mutex);
2239}
2240
2241static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2242 struct intel_crtc *crtc)
2243{
2244 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2245 const struct intel_crtc_state *crtc_state =
2246 intel_atomic_get_new_crtc_state(state, crtc);
2247
2248 if (!crtc_state->wm.need_postvbl_update)
2249 return;
2250
2251 mutex_lock(&dev_priv->wm.wm_mutex);
2252 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2253 vlv_program_watermarks(dev_priv);
2254 mutex_unlock(&dev_priv->wm.wm_mutex);
2255}
2256
2257static void i965_update_wm(struct intel_crtc *unused_crtc)
2258{
2259 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2260 struct intel_crtc *crtc;
2261 int srwm = 1;
2262 int cursor_sr = 16;
2263 bool cxsr_enabled;
2264
2265 /* Calc sr entries for one plane configs */
2266 crtc = single_enabled_crtc(dev_priv);
2267 if (crtc) {
2268 /* self-refresh has much higher latency */
2269 static const int sr_latency_ns = 12000;
2270 const struct drm_display_mode *adjusted_mode =
2271 &crtc->config->hw.adjusted_mode;
2272 const struct drm_framebuffer *fb =
2273 crtc->base.primary->state->fb;
2274 int clock = adjusted_mode->crtc_clock;
2275 int htotal = adjusted_mode->crtc_htotal;
2276 int hdisplay = crtc->config->pipe_src_w;
2277 int cpp = fb->format->cpp[0];
2278 int entries;
2279
2280 entries = intel_wm_method2(clock, htotal,
2281 hdisplay, cpp, sr_latency_ns / 100);
2282 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2283 srwm = I965_FIFO_SIZE - entries;
2284 if (srwm < 0)
2285 srwm = 1;
2286 srwm &= 0x1ff;
2287 drm_dbg_kms(&dev_priv->drm,
2288 "self-refresh entries: %d, wm: %d\n",
2289 entries, srwm);
2290
2291 entries = intel_wm_method2(clock, htotal,
2292 crtc->base.cursor->state->crtc_w, 4,
2293 sr_latency_ns / 100);
2294 entries = DIV_ROUND_UP(entries,
2295 i965_cursor_wm_info.cacheline_size) +
2296 i965_cursor_wm_info.guard_size;
2297
2298 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2299 if (cursor_sr > i965_cursor_wm_info.max_wm)
2300 cursor_sr = i965_cursor_wm_info.max_wm;
2301
2302 drm_dbg_kms(&dev_priv->drm,
2303 "self-refresh watermark: display plane %d "
2304 "cursor %d\n", srwm, cursor_sr);
2305
2306 cxsr_enabled = true;
2307 } else {
2308 cxsr_enabled = false;
2309 /* Turn off self refresh if both pipes are enabled */
2310 intel_set_memory_cxsr(dev_priv, false);
2311 }
2312
2313 drm_dbg_kms(&dev_priv->drm,
2314 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2315 srwm);
2316
2317 /* 965 has limitations... */
2318 I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2319 FW_WM(8, CURSORB) |
2320 FW_WM(8, PLANEB) |
2321 FW_WM(8, PLANEA));
2322 I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2323 FW_WM(8, PLANEC_OLD));
2324 /* update cursor SR watermark */
2325 I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2326
2327 if (cxsr_enabled)
2328 intel_set_memory_cxsr(dev_priv, true);
2329}
2330
2331#undef FW_WM
2332
2333static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2334{
2335 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2336 const struct intel_watermark_params *wm_info;
2337 u32 fwater_lo;
2338 u32 fwater_hi;
2339 int cwm, srwm = 1;
2340 int fifo_size;
2341 int planea_wm, planeb_wm;
2342 struct intel_crtc *crtc, *enabled = NULL;
2343
2344 if (IS_I945GM(dev_priv))
2345 wm_info = &i945_wm_info;
2346 else if (!IS_GEN(dev_priv, 2))
2347 wm_info = &i915_wm_info;
2348 else
2349 wm_info = &i830_a_wm_info;
2350
2351 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2352 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2353 if (intel_crtc_active(crtc)) {
2354 const struct drm_display_mode *adjusted_mode =
2355 &crtc->config->hw.adjusted_mode;
2356 const struct drm_framebuffer *fb =
2357 crtc->base.primary->state->fb;
2358 int cpp;
2359
2360 if (IS_GEN(dev_priv, 2))
2361 cpp = 4;
2362 else
2363 cpp = fb->format->cpp[0];
2364
2365 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2366 wm_info, fifo_size, cpp,
2367 pessimal_latency_ns);
2368 enabled = crtc;
2369 } else {
2370 planea_wm = fifo_size - wm_info->guard_size;
2371 if (planea_wm > (long)wm_info->max_wm)
2372 planea_wm = wm_info->max_wm;
2373 }
2374
2375 if (IS_GEN(dev_priv, 2))
2376 wm_info = &i830_bc_wm_info;
2377
2378 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2379 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2380 if (intel_crtc_active(crtc)) {
2381 const struct drm_display_mode *adjusted_mode =
2382 &crtc->config->hw.adjusted_mode;
2383 const struct drm_framebuffer *fb =
2384 crtc->base.primary->state->fb;
2385 int cpp;
2386
2387 if (IS_GEN(dev_priv, 2))
2388 cpp = 4;
2389 else
2390 cpp = fb->format->cpp[0];
2391
2392 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2393 wm_info, fifo_size, cpp,
2394 pessimal_latency_ns);
2395 if (enabled == NULL)
2396 enabled = crtc;
2397 else
2398 enabled = NULL;
2399 } else {
2400 planeb_wm = fifo_size - wm_info->guard_size;
2401 if (planeb_wm > (long)wm_info->max_wm)
2402 planeb_wm = wm_info->max_wm;
2403 }
2404
2405 drm_dbg_kms(&dev_priv->drm,
2406 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2407
2408 if (IS_I915GM(dev_priv) && enabled) {
2409 struct drm_i915_gem_object *obj;
2410
2411 obj = intel_fb_obj(enabled->base.primary->state->fb);
2412
2413 /* self-refresh seems busted with untiled */
2414 if (!i915_gem_object_is_tiled(obj))
2415 enabled = NULL;
2416 }
2417
2418 /*
2419 * Overlay gets an aggressive default since video jitter is bad.
2420 */
2421 cwm = 2;
2422
2423 /* Play safe and disable self-refresh before adjusting watermarks. */
2424 intel_set_memory_cxsr(dev_priv, false);
2425
2426 /* Calc sr entries for one plane configs */
2427 if (HAS_FW_BLC(dev_priv) && enabled) {
2428 /* self-refresh has much higher latency */
2429 static const int sr_latency_ns = 6000;
2430 const struct drm_display_mode *adjusted_mode =
2431 &enabled->config->hw.adjusted_mode;
2432 const struct drm_framebuffer *fb =
2433 enabled->base.primary->state->fb;
2434 int clock = adjusted_mode->crtc_clock;
2435 int htotal = adjusted_mode->crtc_htotal;
2436 int hdisplay = enabled->config->pipe_src_w;
2437 int cpp;
2438 int entries;
2439
2440 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2441 cpp = 4;
2442 else
2443 cpp = fb->format->cpp[0];
2444
2445 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2446 sr_latency_ns / 100);
2447 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2448 drm_dbg_kms(&dev_priv->drm,
2449 "self-refresh entries: %d\n", entries);
2450 srwm = wm_info->fifo_size - entries;
2451 if (srwm < 0)
2452 srwm = 1;
2453
2454 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2455 I915_WRITE(FW_BLC_SELF,
2456 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2457 else
2458 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2459 }
2460
2461 drm_dbg_kms(&dev_priv->drm,
2462 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2463 planea_wm, planeb_wm, cwm, srwm);
2464
2465 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2466 fwater_hi = (cwm & 0x1f);
2467
2468 /* Set request length to 8 cachelines per fetch */
2469 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2470 fwater_hi = fwater_hi | (1 << 8);
2471
2472 I915_WRITE(FW_BLC, fwater_lo);
2473 I915_WRITE(FW_BLC2, fwater_hi);
2474
2475 if (enabled)
2476 intel_set_memory_cxsr(dev_priv, true);
2477}
2478
2479static void i845_update_wm(struct intel_crtc *unused_crtc)
2480{
2481 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2482 struct intel_crtc *crtc;
2483 const struct drm_display_mode *adjusted_mode;
2484 u32 fwater_lo;
2485 int planea_wm;
2486
2487 crtc = single_enabled_crtc(dev_priv);
2488 if (crtc == NULL)
2489 return;
2490
2491 adjusted_mode = &crtc->config->hw.adjusted_mode;
2492 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2493 &i845_wm_info,
2494 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2495 4, pessimal_latency_ns);
2496 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2497 fwater_lo |= (3<<8) | planea_wm;
2498
2499 drm_dbg_kms(&dev_priv->drm,
2500 "Setting FIFO watermarks - A: %d\n", planea_wm);
2501
2502 I915_WRITE(FW_BLC, fwater_lo);
2503}
2504
2505/* latency must be in 0.1us units. */
2506static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2507 unsigned int cpp,
2508 unsigned int latency)
2509{
2510 unsigned int ret;
2511
2512 ret = intel_wm_method1(pixel_rate, cpp, latency);
2513 ret = DIV_ROUND_UP(ret, 64) + 2;
2514
2515 return ret;
2516}
2517
2518/* latency must be in 0.1us units. */
2519static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2520 unsigned int htotal,
2521 unsigned int width,
2522 unsigned int cpp,
2523 unsigned int latency)
2524{
2525 unsigned int ret;
2526
2527 ret = intel_wm_method2(pixel_rate, htotal,
2528 width, cpp, latency);
2529 ret = DIV_ROUND_UP(ret, 64) + 2;
2530
2531 return ret;
2532}
2533
2534static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2535{
2536 /*
2537 * Neither of these should be possible since this function shouldn't be
2538 * called if the CRTC is off or the plane is invisible. But let's be
2539 * extra paranoid to avoid a potential divide-by-zero if we screw up
2540 * elsewhere in the driver.
2541 */
2542 if (WARN_ON(!cpp))
2543 return 0;
2544 if (WARN_ON(!horiz_pixels))
2545 return 0;
2546
2547 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2548}
2549
2550struct ilk_wm_maximums {
2551 u16 pri;
2552 u16 spr;
2553 u16 cur;
2554 u16 fbc;
2555};
2556
2557/*
2558 * For both WM_PIPE and WM_LP.
2559 * mem_value must be in 0.1us units.
2560 */
2561static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2562 const struct intel_plane_state *plane_state,
2563 u32 mem_value, bool is_lp)
2564{
2565 u32 method1, method2;
2566 int cpp;
2567
2568 if (mem_value == 0)
2569 return U32_MAX;
2570
2571 if (!intel_wm_plane_visible(crtc_state, plane_state))
2572 return 0;
2573
2574 cpp = plane_state->hw.fb->format->cpp[0];
2575
2576 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2577
2578 if (!is_lp)
2579 return method1;
2580
2581 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2582 crtc_state->hw.adjusted_mode.crtc_htotal,
2583 drm_rect_width(&plane_state->uapi.dst),
2584 cpp, mem_value);
2585
2586 return min(method1, method2);
2587}
2588
2589/*
2590 * For both WM_PIPE and WM_LP.
2591 * mem_value must be in 0.1us units.
2592 */
2593static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2594 const struct intel_plane_state *plane_state,
2595 u32 mem_value)
2596{
2597 u32 method1, method2;
2598 int cpp;
2599
2600 if (mem_value == 0)
2601 return U32_MAX;
2602
2603 if (!intel_wm_plane_visible(crtc_state, plane_state))
2604 return 0;
2605
2606 cpp = plane_state->hw.fb->format->cpp[0];
2607
2608 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2609 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2610 crtc_state->hw.adjusted_mode.crtc_htotal,
2611 drm_rect_width(&plane_state->uapi.dst),
2612 cpp, mem_value);
2613 return min(method1, method2);
2614}
2615
2616/*
2617 * For both WM_PIPE and WM_LP.
2618 * mem_value must be in 0.1us units.
2619 */
2620static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2621 const struct intel_plane_state *plane_state,
2622 u32 mem_value)
2623{
2624 int cpp;
2625
2626 if (mem_value == 0)
2627 return U32_MAX;
2628
2629 if (!intel_wm_plane_visible(crtc_state, plane_state))
2630 return 0;
2631
2632 cpp = plane_state->hw.fb->format->cpp[0];
2633
2634 return ilk_wm_method2(crtc_state->pixel_rate,
2635 crtc_state->hw.adjusted_mode.crtc_htotal,
2636 drm_rect_width(&plane_state->uapi.dst),
2637 cpp, mem_value);
2638}
2639
2640/* Only for WM_LP. */
2641static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2642 const struct intel_plane_state *plane_state,
2643 u32 pri_val)
2644{
2645 int cpp;
2646
2647 if (!intel_wm_plane_visible(crtc_state, plane_state))
2648 return 0;
2649
2650 cpp = plane_state->hw.fb->format->cpp[0];
2651
2652 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2653 cpp);
2654}
2655
2656static unsigned int
2657ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2658{
2659 if (INTEL_GEN(dev_priv) >= 8)
2660 return 3072;
2661 else if (INTEL_GEN(dev_priv) >= 7)
2662 return 768;
2663 else
2664 return 512;
2665}
2666
2667static unsigned int
2668ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2669 int level, bool is_sprite)
2670{
2671 if (INTEL_GEN(dev_priv) >= 8)
2672 /* BDW primary/sprite plane watermarks */
2673 return level == 0 ? 255 : 2047;
2674 else if (INTEL_GEN(dev_priv) >= 7)
2675 /* IVB/HSW primary/sprite plane watermarks */
2676 return level == 0 ? 127 : 1023;
2677 else if (!is_sprite)
2678 /* ILK/SNB primary plane watermarks */
2679 return level == 0 ? 127 : 511;
2680 else
2681 /* ILK/SNB sprite plane watermarks */
2682 return level == 0 ? 63 : 255;
2683}
2684
2685static unsigned int
2686ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2687{
2688 if (INTEL_GEN(dev_priv) >= 7)
2689 return level == 0 ? 63 : 255;
2690 else
2691 return level == 0 ? 31 : 63;
2692}
2693
2694static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2695{
2696 if (INTEL_GEN(dev_priv) >= 8)
2697 return 31;
2698 else
2699 return 15;
2700}
2701
2702/* Calculate the maximum primary/sprite plane watermark */
2703static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2704 int level,
2705 const struct intel_wm_config *config,
2706 enum intel_ddb_partitioning ddb_partitioning,
2707 bool is_sprite)
2708{
2709 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2710
2711 /* if sprites aren't enabled, sprites get nothing */
2712 if (is_sprite && !config->sprites_enabled)
2713 return 0;
2714
2715 /* HSW allows LP1+ watermarks even with multiple pipes */
2716 if (level == 0 || config->num_pipes_active > 1) {
2717 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2718
2719 /*
2720 * For some reason the non self refresh
2721 * FIFO size is only half of the self
2722 * refresh FIFO size on ILK/SNB.
2723 */
2724 if (INTEL_GEN(dev_priv) <= 6)
2725 fifo_size /= 2;
2726 }
2727
2728 if (config->sprites_enabled) {
2729 /* level 0 is always calculated with 1:1 split */
2730 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2731 if (is_sprite)
2732 fifo_size *= 5;
2733 fifo_size /= 6;
2734 } else {
2735 fifo_size /= 2;
2736 }
2737 }
2738
2739 /* clamp to max that the registers can hold */
2740 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2741}
2742
2743/* Calculate the maximum cursor plane watermark */
2744static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2745 int level,
2746 const struct intel_wm_config *config)
2747{
2748 /* HSW LP1+ watermarks w/ multiple pipes */
2749 if (level > 0 && config->num_pipes_active > 1)
2750 return 64;
2751
2752 /* otherwise just report max that registers can hold */
2753 return ilk_cursor_wm_reg_max(dev_priv, level);
2754}
2755
2756static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2757 int level,
2758 const struct intel_wm_config *config,
2759 enum intel_ddb_partitioning ddb_partitioning,
2760 struct ilk_wm_maximums *max)
2761{
2762 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2763 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2764 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2765 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2766}
2767
2768static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2769 int level,
2770 struct ilk_wm_maximums *max)
2771{
2772 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2773 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2774 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2775 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2776}
2777
2778static bool ilk_validate_wm_level(int level,
2779 const struct ilk_wm_maximums *max,
2780 struct intel_wm_level *result)
2781{
2782 bool ret;
2783
2784 /* already determined to be invalid? */
2785 if (!result->enable)
2786 return false;
2787
2788 result->enable = result->pri_val <= max->pri &&
2789 result->spr_val <= max->spr &&
2790 result->cur_val <= max->cur;
2791
2792 ret = result->enable;
2793
2794 /*
2795 * HACK until we can pre-compute everything,
2796 * and thus fail gracefully if LP0 watermarks
2797 * are exceeded...
2798 */
2799 if (level == 0 && !result->enable) {
2800 if (result->pri_val > max->pri)
2801 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2802 level, result->pri_val, max->pri);
2803 if (result->spr_val > max->spr)
2804 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2805 level, result->spr_val, max->spr);
2806 if (result->cur_val > max->cur)
2807 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2808 level, result->cur_val, max->cur);
2809
2810 result->pri_val = min_t(u32, result->pri_val, max->pri);
2811 result->spr_val = min_t(u32, result->spr_val, max->spr);
2812 result->cur_val = min_t(u32, result->cur_val, max->cur);
2813 result->enable = true;
2814 }
2815
2816 return ret;
2817}
2818
2819static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2820 const struct intel_crtc *crtc,
2821 int level,
2822 struct intel_crtc_state *crtc_state,
2823 const struct intel_plane_state *pristate,
2824 const struct intel_plane_state *sprstate,
2825 const struct intel_plane_state *curstate,
2826 struct intel_wm_level *result)
2827{
2828 u16 pri_latency = dev_priv->wm.pri_latency[level];
2829 u16 spr_latency = dev_priv->wm.spr_latency[level];
2830 u16 cur_latency = dev_priv->wm.cur_latency[level];
2831
2832 /* WM1+ latency values stored in 0.5us units */
2833 if (level > 0) {
2834 pri_latency *= 5;
2835 spr_latency *= 5;
2836 cur_latency *= 5;
2837 }
2838
2839 if (pristate) {
2840 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2841 pri_latency, level);
2842 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2843 }
2844
2845 if (sprstate)
2846 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2847
2848 if (curstate)
2849 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2850
2851 result->enable = true;
2852}
2853
2854static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2855 u16 wm[8])
2856{
2857 struct intel_uncore *uncore = &dev_priv->uncore;
2858
2859 if (INTEL_GEN(dev_priv) >= 9) {
2860 u32 val;
2861 int ret, i;
2862 int level, max_level = ilk_wm_max_level(dev_priv);
2863
2864 /* read the first set of memory latencies[0:3] */
2865 val = 0; /* data0 to be programmed to 0 for first set */
2866 ret = sandybridge_pcode_read(dev_priv,
2867 GEN9_PCODE_READ_MEM_LATENCY,
2868 &val, NULL);
2869
2870 if (ret) {
2871 drm_err(&dev_priv->drm,
2872 "SKL Mailbox read error = %d\n", ret);
2873 return;
2874 }
2875
2876 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2877 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2878 GEN9_MEM_LATENCY_LEVEL_MASK;
2879 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2880 GEN9_MEM_LATENCY_LEVEL_MASK;
2881 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2882 GEN9_MEM_LATENCY_LEVEL_MASK;
2883
2884 /* read the second set of memory latencies[4:7] */
2885 val = 1; /* data0 to be programmed to 1 for second set */
2886 ret = sandybridge_pcode_read(dev_priv,
2887 GEN9_PCODE_READ_MEM_LATENCY,
2888 &val, NULL);
2889 if (ret) {
2890 drm_err(&dev_priv->drm,
2891 "SKL Mailbox read error = %d\n", ret);
2892 return;
2893 }
2894
2895 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2896 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2897 GEN9_MEM_LATENCY_LEVEL_MASK;
2898 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2899 GEN9_MEM_LATENCY_LEVEL_MASK;
2900 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2901 GEN9_MEM_LATENCY_LEVEL_MASK;
2902
2903 /*
2904 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2905 * need to be disabled. We make sure to sanitize the values out
2906 * of the punit to satisfy this requirement.
2907 */
2908 for (level = 1; level <= max_level; level++) {
2909 if (wm[level] == 0) {
2910 for (i = level + 1; i <= max_level; i++)
2911 wm[i] = 0;
2912 break;
2913 }
2914 }
2915
2916 /*
2917 * WaWmMemoryReadLatency:skl+,glk
2918 *
2919 * punit doesn't take into account the read latency so we need
2920 * to add 2us to the various latency levels we retrieve from the
2921 * punit when level 0 response data us 0us.
2922 */
2923 if (wm[0] == 0) {
2924 wm[0] += 2;
2925 for (level = 1; level <= max_level; level++) {
2926 if (wm[level] == 0)
2927 break;
2928 wm[level] += 2;
2929 }
2930 }
2931
2932 /*
2933 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2934 * If we could not get dimm info enable this WA to prevent from
2935 * any underrun. If not able to get Dimm info assume 16GB dimm
2936 * to avoid any underrun.
2937 */
2938 if (dev_priv->dram_info.is_16gb_dimm)
2939 wm[0] += 1;
2940
2941 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2942 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2943
2944 wm[0] = (sskpd >> 56) & 0xFF;
2945 if (wm[0] == 0)
2946 wm[0] = sskpd & 0xF;
2947 wm[1] = (sskpd >> 4) & 0xFF;
2948 wm[2] = (sskpd >> 12) & 0xFF;
2949 wm[3] = (sskpd >> 20) & 0x1FF;
2950 wm[4] = (sskpd >> 32) & 0x1FF;
2951 } else if (INTEL_GEN(dev_priv) >= 6) {
2952 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2953
2954 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2955 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2956 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2957 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2958 } else if (INTEL_GEN(dev_priv) >= 5) {
2959 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2960
2961 /* ILK primary LP0 latency is 700 ns */
2962 wm[0] = 7;
2963 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2964 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2965 } else {
2966 MISSING_CASE(INTEL_DEVID(dev_priv));
2967 }
2968}
2969
2970static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2971 u16 wm[5])
2972{
2973 /* ILK sprite LP0 latency is 1300 ns */
2974 if (IS_GEN(dev_priv, 5))
2975 wm[0] = 13;
2976}
2977
2978static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2979 u16 wm[5])
2980{
2981 /* ILK cursor LP0 latency is 1300 ns */
2982 if (IS_GEN(dev_priv, 5))
2983 wm[0] = 13;
2984}
2985
2986int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2987{
2988 /* how many WM levels are we expecting */
2989 if (INTEL_GEN(dev_priv) >= 9)
2990 return 7;
2991 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2992 return 4;
2993 else if (INTEL_GEN(dev_priv) >= 6)
2994 return 3;
2995 else
2996 return 2;
2997}
2998
2999static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3000 const char *name,
3001 const u16 wm[8])
3002{
3003 int level, max_level = ilk_wm_max_level(dev_priv);
3004
3005 for (level = 0; level <= max_level; level++) {
3006 unsigned int latency = wm[level];
3007
3008 if (latency == 0) {
3009 drm_dbg_kms(&dev_priv->drm,
3010 "%s WM%d latency not provided\n",
3011 name, level);
3012 continue;
3013 }
3014
3015 /*
3016 * - latencies are in us on gen9.
3017 * - before then, WM1+ latency values are in 0.5us units
3018 */
3019 if (INTEL_GEN(dev_priv) >= 9)
3020 latency *= 10;
3021 else if (level > 0)
3022 latency *= 5;
3023
3024 drm_dbg_kms(&dev_priv->drm,
3025 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3026 wm[level], latency / 10, latency % 10);
3027 }
3028}
3029
3030static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3031 u16 wm[5], u16 min)
3032{
3033 int level, max_level = ilk_wm_max_level(dev_priv);
3034
3035 if (wm[0] >= min)
3036 return false;
3037
3038 wm[0] = max(wm[0], min);
3039 for (level = 1; level <= max_level; level++)
3040 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3041
3042 return true;
3043}
3044
3045static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3046{
3047 bool changed;
3048
3049 /*
3050 * The BIOS provided WM memory latency values are often
3051 * inadequate for high resolution displays. Adjust them.
3052 */
3053 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3054 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3055 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3056
3057 if (!changed)
3058 return;
3059
3060 drm_dbg_kms(&dev_priv->drm,
3061 "WM latency values increased to avoid potential underruns\n");
3062 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3063 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3064 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3065}
3066
3067static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3068{
3069 /*
3070 * On some SNB machines (Thinkpad X220 Tablet at least)
3071 * LP3 usage can cause vblank interrupts to be lost.
3072 * The DEIIR bit will go high but it looks like the CPU
3073 * never gets interrupted.
3074 *
3075 * It's not clear whether other interrupt source could
3076 * be affected or if this is somehow limited to vblank
3077 * interrupts only. To play it safe we disable LP3
3078 * watermarks entirely.
3079 */
3080 if (dev_priv->wm.pri_latency[3] == 0 &&
3081 dev_priv->wm.spr_latency[3] == 0 &&
3082 dev_priv->wm.cur_latency[3] == 0)
3083 return;
3084
3085 dev_priv->wm.pri_latency[3] = 0;
3086 dev_priv->wm.spr_latency[3] = 0;
3087 dev_priv->wm.cur_latency[3] = 0;
3088
3089 drm_dbg_kms(&dev_priv->drm,
3090 "LP3 watermarks disabled due to potential for lost interrupts\n");
3091 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3092 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3093 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3094}
3095
3096static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3097{
3098 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3099
3100 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3101 sizeof(dev_priv->wm.pri_latency));
3102 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3103 sizeof(dev_priv->wm.pri_latency));
3104
3105 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3106 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3107
3108 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3109 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3110 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3111
3112 if (IS_GEN(dev_priv, 6)) {
3113 snb_wm_latency_quirk(dev_priv);
3114 snb_wm_lp3_irq_quirk(dev_priv);
3115 }
3116}
3117
3118static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3119{
3120 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3121 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3122}
3123
3124static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3125 struct intel_pipe_wm *pipe_wm)
3126{
3127 /* LP0 watermark maximums depend on this pipe alone */
3128 const struct intel_wm_config config = {
3129 .num_pipes_active = 1,
3130 .sprites_enabled = pipe_wm->sprites_enabled,
3131 .sprites_scaled = pipe_wm->sprites_scaled,
3132 };
3133 struct ilk_wm_maximums max;
3134
3135 /* LP0 watermarks always use 1/2 DDB partitioning */
3136 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3137
3138 /* At least LP0 must be valid */
3139 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3140 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3141 return false;
3142 }
3143
3144 return true;
3145}
3146
3147/* Compute new watermarks for the pipe */
3148static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3149{
3150 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3151 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3152 struct intel_pipe_wm *pipe_wm;
3153 struct intel_plane *plane;
3154 const struct intel_plane_state *plane_state;
3155 const struct intel_plane_state *pristate = NULL;
3156 const struct intel_plane_state *sprstate = NULL;
3157 const struct intel_plane_state *curstate = NULL;
3158 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3159 struct ilk_wm_maximums max;
3160
3161 pipe_wm = &crtc_state->wm.ilk.optimal;
3162
3163 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3164 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3165 pristate = plane_state;
3166 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3167 sprstate = plane_state;
3168 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3169 curstate = plane_state;
3170 }
3171
3172 pipe_wm->pipe_enabled = crtc_state->hw.active;
3173 if (sprstate) {
3174 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3175 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3176 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3177 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3178 }
3179
3180 usable_level = max_level;
3181
3182 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3183 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3184 usable_level = 1;
3185
3186 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3187 if (pipe_wm->sprites_scaled)
3188 usable_level = 0;
3189
3190 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3191 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3192 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3193
3194 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3195 return -EINVAL;
3196
3197 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3198
3199 for (level = 1; level <= usable_level; level++) {
3200 struct intel_wm_level *wm = &pipe_wm->wm[level];
3201
3202 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3203 pristate, sprstate, curstate, wm);
3204
3205 /*
3206 * Disable any watermark level that exceeds the
3207 * register maximums since such watermarks are
3208 * always invalid.
3209 */
3210 if (!ilk_validate_wm_level(level, &max, wm)) {
3211 memset(wm, 0, sizeof(*wm));
3212 break;
3213 }
3214 }
3215
3216 return 0;
3217}
3218
3219/*
3220 * Build a set of 'intermediate' watermark values that satisfy both the old
3221 * state and the new state. These can be programmed to the hardware
3222 * immediately.
3223 */
3224static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3225{
3226 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3227 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3228 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3229 struct intel_atomic_state *intel_state =
3230 to_intel_atomic_state(newstate->uapi.state);
3231 const struct intel_crtc_state *oldstate =
3232 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3233 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3234 int level, max_level = ilk_wm_max_level(dev_priv);
3235
3236 /*
3237 * Start with the final, target watermarks, then combine with the
3238 * currently active watermarks to get values that are safe both before
3239 * and after the vblank.
3240 */
3241 *a = newstate->wm.ilk.optimal;
3242 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3243 intel_state->skip_intermediate_wm)
3244 return 0;
3245
3246 a->pipe_enabled |= b->pipe_enabled;
3247 a->sprites_enabled |= b->sprites_enabled;
3248 a->sprites_scaled |= b->sprites_scaled;
3249
3250 for (level = 0; level <= max_level; level++) {
3251 struct intel_wm_level *a_wm = &a->wm[level];
3252 const struct intel_wm_level *b_wm = &b->wm[level];
3253
3254 a_wm->enable &= b_wm->enable;
3255 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3256 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3257 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3258 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3259 }
3260
3261 /*
3262 * We need to make sure that these merged watermark values are
3263 * actually a valid configuration themselves. If they're not,
3264 * there's no safe way to transition from the old state to
3265 * the new state, so we need to fail the atomic transaction.
3266 */
3267 if (!ilk_validate_pipe_wm(dev_priv, a))
3268 return -EINVAL;
3269
3270 /*
3271 * If our intermediate WM are identical to the final WM, then we can
3272 * omit the post-vblank programming; only update if it's different.
3273 */
3274 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3275 newstate->wm.need_postvbl_update = true;
3276
3277 return 0;
3278}
3279
3280/*
3281 * Merge the watermarks from all active pipes for a specific level.
3282 */
3283static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3284 int level,
3285 struct intel_wm_level *ret_wm)
3286{
3287 const struct intel_crtc *intel_crtc;
3288
3289 ret_wm->enable = true;
3290
3291 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3292 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3293 const struct intel_wm_level *wm = &active->wm[level];
3294
3295 if (!active->pipe_enabled)
3296 continue;
3297
3298 /*
3299 * The watermark values may have been used in the past,
3300 * so we must maintain them in the registers for some
3301 * time even if the level is now disabled.
3302 */
3303 if (!wm->enable)
3304 ret_wm->enable = false;
3305
3306 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3307 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3308 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3309 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3310 }
3311}
3312
3313/*
3314 * Merge all low power watermarks for all active pipes.
3315 */
3316static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3317 const struct intel_wm_config *config,
3318 const struct ilk_wm_maximums *max,
3319 struct intel_pipe_wm *merged)
3320{
3321 int level, max_level = ilk_wm_max_level(dev_priv);
3322 int last_enabled_level = max_level;
3323
3324 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3325 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3326 config->num_pipes_active > 1)
3327 last_enabled_level = 0;
3328
3329 /* ILK: FBC WM must be disabled always */
3330 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3331
3332 /* merge each WM1+ level */
3333 for (level = 1; level <= max_level; level++) {
3334 struct intel_wm_level *wm = &merged->wm[level];
3335
3336 ilk_merge_wm_level(dev_priv, level, wm);
3337
3338 if (level > last_enabled_level)
3339 wm->enable = false;
3340 else if (!ilk_validate_wm_level(level, max, wm))
3341 /* make sure all following levels get disabled */
3342 last_enabled_level = level - 1;
3343
3344 /*
3345 * The spec says it is preferred to disable
3346 * FBC WMs instead of disabling a WM level.
3347 */
3348 if (wm->fbc_val > max->fbc) {
3349 if (wm->enable)
3350 merged->fbc_wm_enabled = false;
3351 wm->fbc_val = 0;
3352 }
3353 }
3354
3355 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3356 /*
3357 * FIXME this is racy. FBC might get enabled later.
3358 * What we should check here is whether FBC can be
3359 * enabled sometime later.
3360 */
3361 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3362 intel_fbc_is_active(dev_priv)) {
3363 for (level = 2; level <= max_level; level++) {
3364 struct intel_wm_level *wm = &merged->wm[level];
3365
3366 wm->enable = false;
3367 }
3368 }
3369}
3370
3371static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3372{
3373 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3374 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3375}
3376
3377/* The value we need to program into the WM_LPx latency field */
3378static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3379 int level)
3380{
3381 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3382 return 2 * level;
3383 else
3384 return dev_priv->wm.pri_latency[level];
3385}
3386
3387static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3388 const struct intel_pipe_wm *merged,
3389 enum intel_ddb_partitioning partitioning,
3390 struct ilk_wm_values *results)
3391{
3392 struct intel_crtc *intel_crtc;
3393 int level, wm_lp;
3394
3395 results->enable_fbc_wm = merged->fbc_wm_enabled;
3396 results->partitioning = partitioning;
3397
3398 /* LP1+ register values */
3399 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3400 const struct intel_wm_level *r;
3401
3402 level = ilk_wm_lp_to_level(wm_lp, merged);
3403
3404 r = &merged->wm[level];
3405
3406 /*
3407 * Maintain the watermark values even if the level is
3408 * disabled. Doing otherwise could cause underruns.
3409 */
3410 results->wm_lp[wm_lp - 1] =
3411 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3412 (r->pri_val << WM1_LP_SR_SHIFT) |
3413 r->cur_val;
3414
3415 if (r->enable)
3416 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3417
3418 if (INTEL_GEN(dev_priv) >= 8)
3419 results->wm_lp[wm_lp - 1] |=
3420 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3421 else
3422 results->wm_lp[wm_lp - 1] |=
3423 r->fbc_val << WM1_LP_FBC_SHIFT;
3424
3425 /*
3426 * Always set WM1S_LP_EN when spr_val != 0, even if the
3427 * level is disabled. Doing otherwise could cause underruns.
3428 */
3429 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3430 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3431 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3432 } else
3433 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3434 }
3435
3436 /* LP0 register values */
3437 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3438 enum pipe pipe = intel_crtc->pipe;
3439 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3440 const struct intel_wm_level *r = &pipe_wm->wm[0];
3441
3442 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3443 continue;
3444
3445 results->wm_pipe[pipe] =
3446 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3447 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3448 r->cur_val;
3449 }
3450}
3451
3452/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3453 * case both are at the same level. Prefer r1 in case they're the same. */
3454static struct intel_pipe_wm *
3455ilk_find_best_result(struct drm_i915_private *dev_priv,
3456 struct intel_pipe_wm *r1,
3457 struct intel_pipe_wm *r2)
3458{
3459 int level, max_level = ilk_wm_max_level(dev_priv);
3460 int level1 = 0, level2 = 0;
3461
3462 for (level = 1; level <= max_level; level++) {
3463 if (r1->wm[level].enable)
3464 level1 = level;
3465 if (r2->wm[level].enable)
3466 level2 = level;
3467 }
3468
3469 if (level1 == level2) {
3470 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3471 return r2;
3472 else
3473 return r1;
3474 } else if (level1 > level2) {
3475 return r1;
3476 } else {
3477 return r2;
3478 }
3479}
3480
3481/* dirty bits used to track which watermarks need changes */
3482#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3483#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3484#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3485#define WM_DIRTY_FBC (1 << 24)
3486#define WM_DIRTY_DDB (1 << 25)
3487
3488static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3489 const struct ilk_wm_values *old,
3490 const struct ilk_wm_values *new)
3491{
3492 unsigned int dirty = 0;
3493 enum pipe pipe;
3494 int wm_lp;
3495
3496 for_each_pipe(dev_priv, pipe) {
3497 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3498 dirty |= WM_DIRTY_PIPE(pipe);
3499 /* Must disable LP1+ watermarks too */
3500 dirty |= WM_DIRTY_LP_ALL;
3501 }
3502 }
3503
3504 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3505 dirty |= WM_DIRTY_FBC;
3506 /* Must disable LP1+ watermarks too */
3507 dirty |= WM_DIRTY_LP_ALL;
3508 }
3509
3510 if (old->partitioning != new->partitioning) {
3511 dirty |= WM_DIRTY_DDB;
3512 /* Must disable LP1+ watermarks too */
3513 dirty |= WM_DIRTY_LP_ALL;
3514 }
3515
3516 /* LP1+ watermarks already deemed dirty, no need to continue */
3517 if (dirty & WM_DIRTY_LP_ALL)
3518 return dirty;
3519
3520 /* Find the lowest numbered LP1+ watermark in need of an update... */
3521 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3522 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3523 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3524 break;
3525 }
3526
3527 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3528 for (; wm_lp <= 3; wm_lp++)
3529 dirty |= WM_DIRTY_LP(wm_lp);
3530
3531 return dirty;
3532}
3533
3534static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3535 unsigned int dirty)
3536{
3537 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3538 bool changed = false;
3539
3540 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3541 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3542 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3543 changed = true;
3544 }
3545 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3546 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3547 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3548 changed = true;
3549 }
3550 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3551 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3552 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3553 changed = true;
3554 }
3555
3556 /*
3557 * Don't touch WM1S_LP_EN here.
3558 * Doing so could cause underruns.
3559 */
3560
3561 return changed;
3562}
3563
3564/*
3565 * The spec says we shouldn't write when we don't need, because every write
3566 * causes WMs to be re-evaluated, expending some power.
3567 */
3568static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3569 struct ilk_wm_values *results)
3570{
3571 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3572 unsigned int dirty;
3573 u32 val;
3574
3575 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3576 if (!dirty)
3577 return;
3578
3579 _ilk_disable_lp_wm(dev_priv, dirty);
3580
3581 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3582 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3583 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3584 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3585 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3586 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3587
3588 if (dirty & WM_DIRTY_DDB) {
3589 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3590 val = I915_READ(WM_MISC);
3591 if (results->partitioning == INTEL_DDB_PART_1_2)
3592 val &= ~WM_MISC_DATA_PARTITION_5_6;
3593 else
3594 val |= WM_MISC_DATA_PARTITION_5_6;
3595 I915_WRITE(WM_MISC, val);
3596 } else {
3597 val = I915_READ(DISP_ARB_CTL2);
3598 if (results->partitioning == INTEL_DDB_PART_1_2)
3599 val &= ~DISP_DATA_PARTITION_5_6;
3600 else
3601 val |= DISP_DATA_PARTITION_5_6;
3602 I915_WRITE(DISP_ARB_CTL2, val);
3603 }
3604 }
3605
3606 if (dirty & WM_DIRTY_FBC) {
3607 val = I915_READ(DISP_ARB_CTL);
3608 if (results->enable_fbc_wm)
3609 val &= ~DISP_FBC_WM_DIS;
3610 else
3611 val |= DISP_FBC_WM_DIS;
3612 I915_WRITE(DISP_ARB_CTL, val);
3613 }
3614
3615 if (dirty & WM_DIRTY_LP(1) &&
3616 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3617 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3618
3619 if (INTEL_GEN(dev_priv) >= 7) {
3620 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3621 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3622 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3623 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3624 }
3625
3626 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3627 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3628 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3629 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3630 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3631 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3632
3633 dev_priv->wm.hw = *results;
3634}
3635
3636bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3637{
3638 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3639}
3640
3641u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3642{
3643 int i;
3644 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3645 u8 enabled_slices_mask = 0;
3646
3647 for (i = 0; i < max_slices; i++) {
3648 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3649 enabled_slices_mask |= BIT(i);
3650 }
3651
3652 return enabled_slices_mask;
3653}
3654
3655/*
3656 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3657 * so assume we'll always need it in order to avoid underruns.
3658 */
3659static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3660{
3661 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3662}
3663
3664static bool
3665intel_has_sagv(struct drm_i915_private *dev_priv)
3666{
3667 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3668 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3669}
3670
3671static void
3672skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3673{
3674 if (INTEL_GEN(dev_priv) >= 12) {
3675 u32 val = 0;
3676 int ret;
3677
3678 ret = sandybridge_pcode_read(dev_priv,
3679 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3680 &val, NULL);
3681 if (!ret) {
3682 dev_priv->sagv_block_time_us = val;
3683 return;
3684 }
3685
3686 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3687 } else if (IS_GEN(dev_priv, 11)) {
3688 dev_priv->sagv_block_time_us = 10;
3689 return;
3690 } else if (IS_GEN(dev_priv, 10)) {
3691 dev_priv->sagv_block_time_us = 20;
3692 return;
3693 } else if (IS_GEN(dev_priv, 9)) {
3694 dev_priv->sagv_block_time_us = 30;
3695 return;
3696 } else {
3697 MISSING_CASE(INTEL_GEN(dev_priv));
3698 }
3699
3700 /* Default to an unusable block time */
3701 dev_priv->sagv_block_time_us = -1;
3702}
3703
3704/*
3705 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3706 * depending on power and performance requirements. The display engine access
3707 * to system memory is blocked during the adjustment time. Because of the
3708 * blocking time, having this enabled can cause full system hangs and/or pipe
3709 * underruns if we don't meet all of the following requirements:
3710 *
3711 * - <= 1 pipe enabled
3712 * - All planes can enable watermarks for latencies >= SAGV engine block time
3713 * - We're not using an interlaced display configuration
3714 */
3715int
3716intel_enable_sagv(struct drm_i915_private *dev_priv)
3717{
3718 int ret;
3719
3720 if (!intel_has_sagv(dev_priv))
3721 return 0;
3722
3723 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3724 return 0;
3725
3726 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3727 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3728 GEN9_SAGV_ENABLE);
3729
3730 /* We don't need to wait for SAGV when enabling */
3731
3732 /*
3733 * Some skl systems, pre-release machines in particular,
3734 * don't actually have SAGV.
3735 */
3736 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3737 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3738 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3739 return 0;
3740 } else if (ret < 0) {
3741 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3742 return ret;
3743 }
3744
3745 dev_priv->sagv_status = I915_SAGV_ENABLED;
3746 return 0;
3747}
3748
3749int
3750intel_disable_sagv(struct drm_i915_private *dev_priv)
3751{
3752 int ret;
3753
3754 if (!intel_has_sagv(dev_priv))
3755 return 0;
3756
3757 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3758 return 0;
3759
3760 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3761 /* bspec says to keep retrying for at least 1 ms */
3762 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3763 GEN9_SAGV_DISABLE,
3764 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3765 1);
3766 /*
3767 * Some skl systems, pre-release machines in particular,
3768 * don't actually have SAGV.
3769 */
3770 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3771 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3772 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3773 return 0;
3774 } else if (ret < 0) {
3775 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3776 return ret;
3777 }
3778
3779 dev_priv->sagv_status = I915_SAGV_DISABLED;
3780 return 0;
3781}
3782
3783void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3784{
3785 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3786 const struct intel_bw_state *new_bw_state;
3787 const struct intel_bw_state *old_bw_state;
3788 u32 new_mask = 0;
3789
3790 /*
3791 * Just return if we can't control SAGV or don't have it.
3792 * This is different from situation when we have SAGV but just can't
3793 * afford it due to DBuf limitation - in case if SAGV is completely
3794 * disabled in a BIOS, we are not even allowed to send a PCode request,
3795 * as it will throw an error. So have to check it here.
3796 */
3797 if (!intel_has_sagv(dev_priv))
3798 return;
3799
3800 new_bw_state = intel_atomic_get_new_bw_state(state);
3801 if (!new_bw_state)
3802 return;
3803
3804 if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3805 intel_disable_sagv(dev_priv);
3806 return;
3807 }
3808
3809 old_bw_state = intel_atomic_get_old_bw_state(state);
3810 /*
3811 * Nothing to mask
3812 */
3813 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3814 return;
3815
3816 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3817
3818 /*
3819 * If new mask is zero - means there is nothing to mask,
3820 * we can only unmask, which should be done in unmask.
3821 */
3822 if (!new_mask)
3823 return;
3824
3825 /*
3826 * Restrict required qgv points before updating the configuration.
3827 * According to BSpec we can't mask and unmask qgv points at the same
3828 * time. Also masking should be done before updating the configuration
3829 * and unmasking afterwards.
3830 */
3831 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3832}
3833
3834void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3835{
3836 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3837 const struct intel_bw_state *new_bw_state;
3838 const struct intel_bw_state *old_bw_state;
3839 u32 new_mask = 0;
3840
3841 /*
3842 * Just return if we can't control SAGV or don't have it.
3843 * This is different from situation when we have SAGV but just can't
3844 * afford it due to DBuf limitation - in case if SAGV is completely
3845 * disabled in a BIOS, we are not even allowed to send a PCode request,
3846 * as it will throw an error. So have to check it here.
3847 */
3848 if (!intel_has_sagv(dev_priv))
3849 return;
3850
3851 new_bw_state = intel_atomic_get_new_bw_state(state);
3852 if (!new_bw_state)
3853 return;
3854
3855 if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3856 intel_enable_sagv(dev_priv);
3857 return;
3858 }
3859
3860 old_bw_state = intel_atomic_get_old_bw_state(state);
3861 /*
3862 * Nothing to unmask
3863 */
3864 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3865 return;
3866
3867 new_mask = new_bw_state->qgv_points_mask;
3868
3869 /*
3870 * Allow required qgv points after updating the configuration.
3871 * According to BSpec we can't mask and unmask qgv points at the same
3872 * time. Also masking should be done before updating the configuration
3873 * and unmasking afterwards.
3874 */
3875 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3876}
3877
3878static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3879{
3880 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3881 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3882 struct intel_plane *plane;
3883 const struct intel_plane_state *plane_state;
3884 int level, latency;
3885
3886 if (!intel_has_sagv(dev_priv))
3887 return false;
3888
3889 if (!crtc_state->hw.active)
3890 return true;
3891
3892 if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3893 return false;
3894
3895 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3896 const struct skl_plane_wm *wm =
3897 &crtc_state->wm.skl.optimal.planes[plane->id];
3898
3899 /* Skip this plane if it's not enabled */
3900 if (!wm->wm[0].plane_en)
3901 continue;
3902
3903 /* Find the highest enabled wm level for this plane */
3904 for (level = ilk_wm_max_level(dev_priv);
3905 !wm->wm[level].plane_en; --level)
3906 { }
3907
3908 latency = dev_priv->wm.skl_latency[level];
3909
3910 if (skl_needs_memory_bw_wa(dev_priv) &&
3911 plane_state->uapi.fb->modifier ==
3912 I915_FORMAT_MOD_X_TILED)
3913 latency += 15;
3914
3915 /*
3916 * If any of the planes on this pipe don't enable wm levels that
3917 * incur memory latencies higher than sagv_block_time_us we
3918 * can't enable SAGV.
3919 */
3920 if (latency < dev_priv->sagv_block_time_us)
3921 return false;
3922 }
3923
3924 return true;
3925}
3926
3927static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3928{
3929 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3930 enum plane_id plane_id;
3931
3932 if (!crtc_state->hw.active)
3933 return true;
3934
3935 for_each_plane_id_on_crtc(crtc, plane_id) {
3936 const struct skl_ddb_entry *plane_alloc =
3937 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3938 const struct skl_plane_wm *wm =
3939 &crtc_state->wm.skl.optimal.planes[plane_id];
3940
3941 if (skl_ddb_entry_size(plane_alloc) < wm->sagv_wm0.min_ddb_alloc)
3942 return false;
3943 }
3944
3945 return true;
3946}
3947
3948static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3949{
3950 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3951 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3952
3953 if (INTEL_GEN(dev_priv) >= 12)
3954 return tgl_crtc_can_enable_sagv(crtc_state);
3955 else
3956 return skl_crtc_can_enable_sagv(crtc_state);
3957}
3958
3959bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3960 const struct intel_bw_state *bw_state)
3961{
3962 if (INTEL_GEN(dev_priv) < 11 &&
3963 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3964 return false;
3965
3966 return bw_state->pipe_sagv_reject == 0;
3967}
3968
3969static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3970{
3971 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3972 int ret;
3973 struct intel_crtc *crtc;
3974 struct intel_crtc_state *new_crtc_state;
3975 struct intel_bw_state *new_bw_state = NULL;
3976 const struct intel_bw_state *old_bw_state = NULL;
3977 int i;
3978
3979 for_each_new_intel_crtc_in_state(state, crtc,
3980 new_crtc_state, i) {
3981 new_bw_state = intel_atomic_get_bw_state(state);
3982 if (IS_ERR(new_bw_state))
3983 return PTR_ERR(new_bw_state);
3984
3985 old_bw_state = intel_atomic_get_old_bw_state(state);
3986
3987 if (intel_crtc_can_enable_sagv(new_crtc_state))
3988 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3989 else
3990 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3991 }
3992
3993 if (!new_bw_state)
3994 return 0;
3995
3996 new_bw_state->active_pipes =
3997 intel_calc_active_pipes(state, old_bw_state->active_pipes);
3998
3999 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4000 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4001 if (ret)
4002 return ret;
4003 }
4004
4005 for_each_new_intel_crtc_in_state(state, crtc,
4006 new_crtc_state, i) {
4007 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4008
4009 /*
4010 * We store use_sagv_wm in the crtc state rather than relying on
4011 * that bw state since we have no convenient way to get at the
4012 * latter from the plane commit hooks (especially in the legacy
4013 * cursor case)
4014 */
4015 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4016 intel_can_enable_sagv(dev_priv, new_bw_state);
4017 }
4018
4019 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4020 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4021 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4022 if (ret)
4023 return ret;
4024 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4025 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4026 if (ret)
4027 return ret;
4028 }
4029
4030 return 0;
4031}
4032
4033/*
4034 * Calculate initial DBuf slice offset, based on slice size
4035 * and mask(i.e if slice size is 1024 and second slice is enabled
4036 * offset would be 1024)
4037 */
4038static unsigned int
4039icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
4040 u32 slice_size,
4041 u32 ddb_size)
4042{
4043 unsigned int offset = 0;
4044
4045 if (!dbuf_slice_mask)
4046 return 0;
4047
4048 offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
4049
4050 WARN_ON(offset >= ddb_size);
4051 return offset;
4052}
4053
4054u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
4055{
4056 u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4057 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4058
4059 if (INTEL_GEN(dev_priv) < 11)
4060 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4061
4062 return ddb_size;
4063}
4064
4065u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4066 const struct skl_ddb_entry *entry)
4067{
4068 u32 slice_mask = 0;
4069 u16 ddb_size = intel_get_ddb_size(dev_priv);
4070 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4071 u16 slice_size = ddb_size / num_supported_slices;
4072 u16 start_slice;
4073 u16 end_slice;
4074
4075 if (!skl_ddb_entry_size(entry))
4076 return 0;
4077
4078 start_slice = entry->start / slice_size;
4079 end_slice = (entry->end - 1) / slice_size;
4080
4081 /*
4082 * Per plane DDB entry can in a really worst case be on multiple slices
4083 * but single entry is anyway contigious.
4084 */
4085 while (start_slice <= end_slice) {
4086 slice_mask |= BIT(start_slice);
4087 start_slice++;
4088 }
4089
4090 return slice_mask;
4091}
4092
4093static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4094 u8 active_pipes);
4095
4096static int
4097skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
4098 const struct intel_crtc_state *crtc_state,
4099 const u64 total_data_rate,
4100 struct skl_ddb_entry *alloc, /* out */
4101 int *num_active /* out */)
4102{
4103 struct drm_atomic_state *state = crtc_state->uapi.state;
4104 struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4105 struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
4106 const struct intel_crtc *crtc;
4107 u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
4108 enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
4109 struct intel_dbuf_state *new_dbuf_state =
4110 intel_atomic_get_new_dbuf_state(intel_state);
4111 const struct intel_dbuf_state *old_dbuf_state =
4112 intel_atomic_get_old_dbuf_state(intel_state);
4113 u8 active_pipes = new_dbuf_state->active_pipes;
4114 u16 ddb_size;
4115 u32 ddb_range_size;
4116 u32 i;
4117 u32 dbuf_slice_mask;
4118 u32 offset;
4119 u32 slice_size;
4120 u32 total_slice_mask;
4121 u32 start, end;
4122 int ret;
4123
4124 *num_active = hweight8(active_pipes);
4125
4126 if (!crtc_state->hw.active) {
4127 alloc->start = 0;
4128 alloc->end = 0;
4129 return 0;
4130 }
4131
4132 ddb_size = intel_get_ddb_size(dev_priv);
4133
4134 slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4135
4136 /*
4137 * If the state doesn't change the active CRTC's or there is no
4138 * modeset request, then there's no need to recalculate;
4139 * the existing pipe allocation limits should remain unchanged.
4140 * Note that we're safe from racing commits since any racing commit
4141 * that changes the active CRTC list or do modeset would need to
4142 * grab _all_ crtc locks, including the one we currently hold.
4143 */
4144 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes &&
4145 !dev_priv->wm.distrust_bios_wm) {
4146 /*
4147 * alloc may be cleared by clear_intel_crtc_state,
4148 * copy from old state to be sure
4149 *
4150 * FIXME get rid of this mess
4151 */
4152 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
4153 return 0;
4154 }
4155
4156 /*
4157 * Get allowed DBuf slices for correspondent pipe and platform.
4158 */
4159 dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
4160
4161 /*
4162 * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
4163 * and slice size is 1024, the offset would be 1024
4164 */
4165 offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
4166 slice_size, ddb_size);
4167
4168 /*
4169 * Figure out total size of allowed DBuf slices, which is basically
4170 * a number of allowed slices for that pipe multiplied by slice size.
4171 * Inside of this
4172 * range ddb entries are still allocated in proportion to display width.
4173 */
4174 ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
4175
4176 /*
4177 * Watermark/ddb requirement highly depends upon width of the
4178 * framebuffer, So instead of allocating DDB equally among pipes
4179 * distribute DDB based on resolution/width of the display.
4180 */
4181 total_slice_mask = dbuf_slice_mask;
4182 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
4183 const struct drm_display_mode *adjusted_mode =
4184 &crtc_state->hw.adjusted_mode;
4185 enum pipe pipe = crtc->pipe;
4186 int hdisplay, vdisplay;
4187 u32 pipe_dbuf_slice_mask;
4188
4189 if (!crtc_state->hw.active)
4190 continue;
4191
4192 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
4193 active_pipes);
4194
4195 /*
4196 * According to BSpec pipe can share one dbuf slice with another
4197 * pipes or pipe can use multiple dbufs, in both cases we
4198 * account for other pipes only if they have exactly same mask.
4199 * However we need to account how many slices we should enable
4200 * in total.
4201 */
4202 total_slice_mask |= pipe_dbuf_slice_mask;
4203
4204 /*
4205 * Do not account pipes using other slice sets
4206 * luckily as of current BSpec slice sets do not partially
4207 * intersect(pipes share either same one slice or same slice set
4208 * i.e no partial intersection), so it is enough to check for
4209 * equality for now.
4210 */
4211 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
4212 continue;
4213
4214 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
4215
4216 total_width_in_range += hdisplay;
4217
4218 if (pipe < for_pipe)
4219 width_before_pipe_in_range += hdisplay;
4220 else if (pipe == for_pipe)
4221 pipe_width = hdisplay;
4222 }
4223
4224 /*
4225 * FIXME: For now we always enable slice S1 as per
4226 * the Bspec display initialization sequence.
4227 */
4228 new_dbuf_state->enabled_slices = total_slice_mask | BIT(DBUF_S1);
4229
4230 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4231 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4232 if (ret)
4233 return ret;
4234 }
4235
4236 start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
4237 end = ddb_range_size *
4238 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
4239
4240 alloc->start = offset + start;
4241 alloc->end = offset + end;
4242
4243 drm_dbg_kms(&dev_priv->drm,
4244 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
4245 for_crtc->base.id, for_crtc->name,
4246 dbuf_slice_mask, alloc->start, alloc->end, active_pipes);
4247
4248 return 0;
4249}
4250
4251static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4252 int width, const struct drm_format_info *format,
4253 u64 modifier, unsigned int rotation,
4254 u32 plane_pixel_rate, struct skl_wm_params *wp,
4255 int color_plane);
4256static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4257 int level,
4258 unsigned int latency,
4259 const struct skl_wm_params *wp,
4260 const struct skl_wm_level *result_prev,
4261 struct skl_wm_level *result /* out */);
4262
4263static unsigned int
4264skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4265 int num_active)
4266{
4267 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4268 int level, max_level = ilk_wm_max_level(dev_priv);
4269 struct skl_wm_level wm = {};
4270 int ret, min_ddb_alloc = 0;
4271 struct skl_wm_params wp;
4272
4273 ret = skl_compute_wm_params(crtc_state, 256,
4274 drm_format_info(DRM_FORMAT_ARGB8888),
4275 DRM_FORMAT_MOD_LINEAR,
4276 DRM_MODE_ROTATE_0,
4277 crtc_state->pixel_rate, &wp, 0);
4278 drm_WARN_ON(&dev_priv->drm, ret);
4279
4280 for (level = 0; level <= max_level; level++) {
4281 unsigned int latency = dev_priv->wm.skl_latency[level];
4282
4283 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4284 if (wm.min_ddb_alloc == U16_MAX)
4285 break;
4286
4287 min_ddb_alloc = wm.min_ddb_alloc;
4288 }
4289
4290 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4291}
4292
4293static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4294 struct skl_ddb_entry *entry, u32 reg)
4295{
4296
4297 entry->start = reg & DDB_ENTRY_MASK;
4298 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4299
4300 if (entry->end)
4301 entry->end += 1;
4302}
4303
4304static void
4305skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4306 const enum pipe pipe,
4307 const enum plane_id plane_id,
4308 struct skl_ddb_entry *ddb_y,
4309 struct skl_ddb_entry *ddb_uv)
4310{
4311 u32 val, val2;
4312 u32 fourcc = 0;
4313
4314 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4315 if (plane_id == PLANE_CURSOR) {
4316 val = I915_READ(CUR_BUF_CFG(pipe));
4317 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4318 return;
4319 }
4320
4321 val = I915_READ(PLANE_CTL(pipe, plane_id));
4322
4323 /* No DDB allocated for disabled planes */
4324 if (val & PLANE_CTL_ENABLE)
4325 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4326 val & PLANE_CTL_ORDER_RGBX,
4327 val & PLANE_CTL_ALPHA_MASK);
4328
4329 if (INTEL_GEN(dev_priv) >= 11) {
4330 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4331 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4332 } else {
4333 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4334 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4335
4336 if (fourcc &&
4337 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4338 swap(val, val2);
4339
4340 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4341 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4342 }
4343}
4344
4345void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4346 struct skl_ddb_entry *ddb_y,
4347 struct skl_ddb_entry *ddb_uv)
4348{
4349 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4350 enum intel_display_power_domain power_domain;
4351 enum pipe pipe = crtc->pipe;
4352 intel_wakeref_t wakeref;
4353 enum plane_id plane_id;
4354
4355 power_domain = POWER_DOMAIN_PIPE(pipe);
4356 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4357 if (!wakeref)
4358 return;
4359
4360 for_each_plane_id_on_crtc(crtc, plane_id)
4361 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4362 plane_id,
4363 &ddb_y[plane_id],
4364 &ddb_uv[plane_id]);
4365
4366 intel_display_power_put(dev_priv, power_domain, wakeref);
4367}
4368
4369/*
4370 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4371 * The bspec defines downscale amount as:
4372 *
4373 * """
4374 * Horizontal down scale amount = maximum[1, Horizontal source size /
4375 * Horizontal destination size]
4376 * Vertical down scale amount = maximum[1, Vertical source size /
4377 * Vertical destination size]
4378 * Total down scale amount = Horizontal down scale amount *
4379 * Vertical down scale amount
4380 * """
4381 *
4382 * Return value is provided in 16.16 fixed point form to retain fractional part.
4383 * Caller should take care of dividing & rounding off the value.
4384 */
4385static uint_fixed_16_16_t
4386skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4387 const struct intel_plane_state *plane_state)
4388{
4389 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4390 u32 src_w, src_h, dst_w, dst_h;
4391 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4392 uint_fixed_16_16_t downscale_h, downscale_w;
4393
4394 if (drm_WARN_ON(&dev_priv->drm,
4395 !intel_wm_plane_visible(crtc_state, plane_state)))
4396 return u32_to_fixed16(0);
4397
4398 /*
4399 * Src coordinates are already rotated by 270 degrees for
4400 * the 90/270 degree plane rotation cases (to match the
4401 * GTT mapping), hence no need to account for rotation here.
4402 *
4403 * n.b., src is 16.16 fixed point, dst is whole integer.
4404 */
4405 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4406 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4407 dst_w = drm_rect_width(&plane_state->uapi.dst);
4408 dst_h = drm_rect_height(&plane_state->uapi.dst);
4409
4410 fp_w_ratio = div_fixed16(src_w, dst_w);
4411 fp_h_ratio = div_fixed16(src_h, dst_h);
4412 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4413 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4414
4415 return mul_fixed16(downscale_w, downscale_h);
4416}
4417
4418struct dbuf_slice_conf_entry {
4419 u8 active_pipes;
4420 u8 dbuf_mask[I915_MAX_PIPES];
4421};
4422
4423/*
4424 * Table taken from Bspec 12716
4425 * Pipes do have some preferred DBuf slice affinity,
4426 * plus there are some hardcoded requirements on how
4427 * those should be distributed for multipipe scenarios.
4428 * For more DBuf slices algorithm can get even more messy
4429 * and less readable, so decided to use a table almost
4430 * as is from BSpec itself - that way it is at least easier
4431 * to compare, change and check.
4432 */
4433static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4434/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4435{
4436 {
4437 .active_pipes = BIT(PIPE_A),
4438 .dbuf_mask = {
4439 [PIPE_A] = BIT(DBUF_S1),
4440 },
4441 },
4442 {
4443 .active_pipes = BIT(PIPE_B),
4444 .dbuf_mask = {
4445 [PIPE_B] = BIT(DBUF_S1),
4446 },
4447 },
4448 {
4449 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4450 .dbuf_mask = {
4451 [PIPE_A] = BIT(DBUF_S1),
4452 [PIPE_B] = BIT(DBUF_S2),
4453 },
4454 },
4455 {
4456 .active_pipes = BIT(PIPE_C),
4457 .dbuf_mask = {
4458 [PIPE_C] = BIT(DBUF_S2),
4459 },
4460 },
4461 {
4462 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4463 .dbuf_mask = {
4464 [PIPE_A] = BIT(DBUF_S1),
4465 [PIPE_C] = BIT(DBUF_S2),
4466 },
4467 },
4468 {
4469 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4470 .dbuf_mask = {
4471 [PIPE_B] = BIT(DBUF_S1),
4472 [PIPE_C] = BIT(DBUF_S2),
4473 },
4474 },
4475 {
4476 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4477 .dbuf_mask = {
4478 [PIPE_A] = BIT(DBUF_S1),
4479 [PIPE_B] = BIT(DBUF_S1),
4480 [PIPE_C] = BIT(DBUF_S2),
4481 },
4482 },
4483 {}
4484};
4485
4486/*
4487 * Table taken from Bspec 49255
4488 * Pipes do have some preferred DBuf slice affinity,
4489 * plus there are some hardcoded requirements on how
4490 * those should be distributed for multipipe scenarios.
4491 * For more DBuf slices algorithm can get even more messy
4492 * and less readable, so decided to use a table almost
4493 * as is from BSpec itself - that way it is at least easier
4494 * to compare, change and check.
4495 */
4496static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4497/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4498{
4499 {
4500 .active_pipes = BIT(PIPE_A),
4501 .dbuf_mask = {
4502 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4503 },
4504 },
4505 {
4506 .active_pipes = BIT(PIPE_B),
4507 .dbuf_mask = {
4508 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4509 },
4510 },
4511 {
4512 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4513 .dbuf_mask = {
4514 [PIPE_A] = BIT(DBUF_S2),
4515 [PIPE_B] = BIT(DBUF_S1),
4516 },
4517 },
4518 {
4519 .active_pipes = BIT(PIPE_C),
4520 .dbuf_mask = {
4521 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4522 },
4523 },
4524 {
4525 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4526 .dbuf_mask = {
4527 [PIPE_A] = BIT(DBUF_S1),
4528 [PIPE_C] = BIT(DBUF_S2),
4529 },
4530 },
4531 {
4532 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4533 .dbuf_mask = {
4534 [PIPE_B] = BIT(DBUF_S1),
4535 [PIPE_C] = BIT(DBUF_S2),
4536 },
4537 },
4538 {
4539 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4540 .dbuf_mask = {
4541 [PIPE_A] = BIT(DBUF_S1),
4542 [PIPE_B] = BIT(DBUF_S1),
4543 [PIPE_C] = BIT(DBUF_S2),
4544 },
4545 },
4546 {
4547 .active_pipes = BIT(PIPE_D),
4548 .dbuf_mask = {
4549 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4550 },
4551 },
4552 {
4553 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4554 .dbuf_mask = {
4555 [PIPE_A] = BIT(DBUF_S1),
4556 [PIPE_D] = BIT(DBUF_S2),
4557 },
4558 },
4559 {
4560 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4561 .dbuf_mask = {
4562 [PIPE_B] = BIT(DBUF_S1),
4563 [PIPE_D] = BIT(DBUF_S2),
4564 },
4565 },
4566 {
4567 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4568 .dbuf_mask = {
4569 [PIPE_A] = BIT(DBUF_S1),
4570 [PIPE_B] = BIT(DBUF_S1),
4571 [PIPE_D] = BIT(DBUF_S2),
4572 },
4573 },
4574 {
4575 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4576 .dbuf_mask = {
4577 [PIPE_C] = BIT(DBUF_S1),
4578 [PIPE_D] = BIT(DBUF_S2),
4579 },
4580 },
4581 {
4582 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4583 .dbuf_mask = {
4584 [PIPE_A] = BIT(DBUF_S1),
4585 [PIPE_C] = BIT(DBUF_S2),
4586 [PIPE_D] = BIT(DBUF_S2),
4587 },
4588 },
4589 {
4590 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4591 .dbuf_mask = {
4592 [PIPE_B] = BIT(DBUF_S1),
4593 [PIPE_C] = BIT(DBUF_S2),
4594 [PIPE_D] = BIT(DBUF_S2),
4595 },
4596 },
4597 {
4598 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4599 .dbuf_mask = {
4600 [PIPE_A] = BIT(DBUF_S1),
4601 [PIPE_B] = BIT(DBUF_S1),
4602 [PIPE_C] = BIT(DBUF_S2),
4603 [PIPE_D] = BIT(DBUF_S2),
4604 },
4605 },
4606 {}
4607};
4608
4609static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4610 const struct dbuf_slice_conf_entry *dbuf_slices)
4611{
4612 int i;
4613
4614 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4615 if (dbuf_slices[i].active_pipes == active_pipes)
4616 return dbuf_slices[i].dbuf_mask[pipe];
4617 }
4618 return 0;
4619}
4620
4621/*
4622 * This function finds an entry with same enabled pipe configuration and
4623 * returns correspondent DBuf slice mask as stated in BSpec for particular
4624 * platform.
4625 */
4626static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4627{
4628 /*
4629 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4630 * required calculating "pipe ratio" in order to determine
4631 * if one or two slices can be used for single pipe configurations
4632 * as additional constraint to the existing table.
4633 * However based on recent info, it should be not "pipe ratio"
4634 * but rather ratio between pixel_rate and cdclk with additional
4635 * constants, so for now we are using only table until this is
4636 * clarified. Also this is the reason why crtc_state param is
4637 * still here - we will need it once those additional constraints
4638 * pop up.
4639 */
4640 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4641}
4642
4643static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4644{
4645 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4646}
4647
4648static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4649 u8 active_pipes)
4650{
4651 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4652 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4653 enum pipe pipe = crtc->pipe;
4654
4655 if (IS_GEN(dev_priv, 12))
4656 return tgl_compute_dbuf_slices(pipe, active_pipes);
4657 else if (IS_GEN(dev_priv, 11))
4658 return icl_compute_dbuf_slices(pipe, active_pipes);
4659 /*
4660 * For anything else just return one slice yet.
4661 * Should be extended for other platforms.
4662 */
4663 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4664}
4665
4666static u64
4667skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4668 const struct intel_plane_state *plane_state,
4669 int color_plane)
4670{
4671 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4672 const struct drm_framebuffer *fb = plane_state->hw.fb;
4673 u32 data_rate;
4674 u32 width = 0, height = 0;
4675 uint_fixed_16_16_t down_scale_amount;
4676 u64 rate;
4677
4678 if (!plane_state->uapi.visible)
4679 return 0;
4680
4681 if (plane->id == PLANE_CURSOR)
4682 return 0;
4683
4684 if (color_plane == 1 &&
4685 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4686 return 0;
4687
4688 /*
4689 * Src coordinates are already rotated by 270 degrees for
4690 * the 90/270 degree plane rotation cases (to match the
4691 * GTT mapping), hence no need to account for rotation here.
4692 */
4693 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4694 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4695
4696 /* UV plane does 1/2 pixel sub-sampling */
4697 if (color_plane == 1) {
4698 width /= 2;
4699 height /= 2;
4700 }
4701
4702 data_rate = width * height;
4703
4704 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4705
4706 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4707
4708 rate *= fb->format->cpp[color_plane];
4709 return rate;
4710}
4711
4712static u64
4713skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4714 u64 *plane_data_rate,
4715 u64 *uv_plane_data_rate)
4716{
4717 struct intel_plane *plane;
4718 const struct intel_plane_state *plane_state;
4719 u64 total_data_rate = 0;
4720
4721 /* Calculate and cache data rate for each plane */
4722 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4723 enum plane_id plane_id = plane->id;
4724 u64 rate;
4725
4726 /* packed/y */
4727 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4728 plane_data_rate[plane_id] = rate;
4729 total_data_rate += rate;
4730
4731 /* uv-plane */
4732 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4733 uv_plane_data_rate[plane_id] = rate;
4734 total_data_rate += rate;
4735 }
4736
4737 return total_data_rate;
4738}
4739
4740static u64
4741icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4742 u64 *plane_data_rate)
4743{
4744 struct intel_plane *plane;
4745 const struct intel_plane_state *plane_state;
4746 u64 total_data_rate = 0;
4747
4748 /* Calculate and cache data rate for each plane */
4749 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4750 enum plane_id plane_id = plane->id;
4751 u64 rate;
4752
4753 if (!plane_state->planar_linked_plane) {
4754 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4755 plane_data_rate[plane_id] = rate;
4756 total_data_rate += rate;
4757 } else {
4758 enum plane_id y_plane_id;
4759
4760 /*
4761 * The slave plane might not iterate in
4762 * intel_atomic_crtc_state_for_each_plane_state(),
4763 * and needs the master plane state which may be
4764 * NULL if we try get_new_plane_state(), so we
4765 * always calculate from the master.
4766 */
4767 if (plane_state->planar_slave)
4768 continue;
4769
4770 /* Y plane rate is calculated on the slave */
4771 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4772 y_plane_id = plane_state->planar_linked_plane->id;
4773 plane_data_rate[y_plane_id] = rate;
4774 total_data_rate += rate;
4775
4776 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4777 plane_data_rate[plane_id] = rate;
4778 total_data_rate += rate;
4779 }
4780 }
4781
4782 return total_data_rate;
4783}
4784
4785static const struct skl_wm_level *
4786skl_plane_wm_level(const struct intel_crtc_state *crtc_state,
4787 enum plane_id plane_id,
4788 int level)
4789{
4790 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
4791 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4792
4793 if (level == 0 && pipe_wm->use_sagv_wm)
4794 return &wm->sagv_wm0;
4795
4796 return &wm->wm[level];
4797}
4798
4799static int
4800skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4801{
4802 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4803 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4804 struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4805 u16 alloc_size, start = 0;
4806 u16 total[I915_MAX_PLANES] = {};
4807 u16 uv_total[I915_MAX_PLANES] = {};
4808 u64 total_data_rate;
4809 enum plane_id plane_id;
4810 int num_active;
4811 u64 plane_data_rate[I915_MAX_PLANES] = {};
4812 u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4813 u32 blocks;
4814 int level;
4815 int ret;
4816
4817 /* Clear the partitioning for disabled planes. */
4818 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4819 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4820
4821 if (!crtc_state->hw.active) {
4822 struct intel_atomic_state *state =
4823 to_intel_atomic_state(crtc_state->uapi.state);
4824 struct intel_dbuf_state *new_dbuf_state =
4825 intel_atomic_get_new_dbuf_state(state);
4826 const struct intel_dbuf_state *old_dbuf_state =
4827 intel_atomic_get_old_dbuf_state(state);
4828
4829 /*
4830 * FIXME hack to make sure we compute this sensibly when
4831 * turning off all the pipes. Otherwise we leave it at
4832 * whatever we had previously, and then runtime PM will
4833 * mess it up by turning off all but S1. Remove this
4834 * once the dbuf state computation flow becomes sane.
4835 */
4836 if (new_dbuf_state->active_pipes == 0) {
4837 new_dbuf_state->enabled_slices = BIT(DBUF_S1);
4838
4839 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
4840 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
4841 if (ret)
4842 return ret;
4843 }
4844 }
4845
4846 alloc->start = alloc->end = 0;
4847 return 0;
4848 }
4849
4850 if (INTEL_GEN(dev_priv) >= 11)
4851 total_data_rate =
4852 icl_get_total_relative_data_rate(crtc_state,
4853 plane_data_rate);
4854 else
4855 total_data_rate =
4856 skl_get_total_relative_data_rate(crtc_state,
4857 plane_data_rate,
4858 uv_plane_data_rate);
4859
4860 ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state,
4861 total_data_rate,
4862 alloc, &num_active);
4863 if (ret)
4864 return ret;
4865
4866 alloc_size = skl_ddb_entry_size(alloc);
4867 if (alloc_size == 0)
4868 return 0;
4869
4870 /* Allocate fixed number of blocks for cursor. */
4871 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4872 alloc_size -= total[PLANE_CURSOR];
4873 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4874 alloc->end - total[PLANE_CURSOR];
4875 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4876
4877 if (total_data_rate == 0)
4878 return 0;
4879
4880 /*
4881 * Find the highest watermark level for which we can satisfy the block
4882 * requirement of active planes.
4883 */
4884 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4885 blocks = 0;
4886 for_each_plane_id_on_crtc(crtc, plane_id) {
4887 const struct skl_plane_wm *wm =
4888 &crtc_state->wm.skl.optimal.planes[plane_id];
4889
4890 if (plane_id == PLANE_CURSOR) {
4891 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4892 drm_WARN_ON(&dev_priv->drm,
4893 wm->wm[level].min_ddb_alloc != U16_MAX);
4894 blocks = U32_MAX;
4895 break;
4896 }
4897 continue;
4898 }
4899
4900 blocks += wm->wm[level].min_ddb_alloc;
4901 blocks += wm->uv_wm[level].min_ddb_alloc;
4902 }
4903
4904 if (blocks <= alloc_size) {
4905 alloc_size -= blocks;
4906 break;
4907 }
4908 }
4909
4910 if (level < 0) {
4911 drm_dbg_kms(&dev_priv->drm,
4912 "Requested display configuration exceeds system DDB limitations");
4913 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4914 blocks, alloc_size);
4915 return -EINVAL;
4916 }
4917
4918 /*
4919 * Grant each plane the blocks it requires at the highest achievable
4920 * watermark level, plus an extra share of the leftover blocks
4921 * proportional to its relative data rate.
4922 */
4923 for_each_plane_id_on_crtc(crtc, plane_id) {
4924 const struct skl_plane_wm *wm =
4925 &crtc_state->wm.skl.optimal.planes[plane_id];
4926 u64 rate;
4927 u16 extra;
4928
4929 if (plane_id == PLANE_CURSOR)
4930 continue;
4931
4932 /*
4933 * We've accounted for all active planes; remaining planes are
4934 * all disabled.
4935 */
4936 if (total_data_rate == 0)
4937 break;
4938
4939 rate = plane_data_rate[plane_id];
4940 extra = min_t(u16, alloc_size,
4941 DIV64_U64_ROUND_UP(alloc_size * rate,
4942 total_data_rate));
4943 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4944 alloc_size -= extra;
4945 total_data_rate -= rate;
4946
4947 if (total_data_rate == 0)
4948 break;
4949
4950 rate = uv_plane_data_rate[plane_id];
4951 extra = min_t(u16, alloc_size,
4952 DIV64_U64_ROUND_UP(alloc_size * rate,
4953 total_data_rate));
4954 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4955 alloc_size -= extra;
4956 total_data_rate -= rate;
4957 }
4958 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4959
4960 /* Set the actual DDB start/end points for each plane */
4961 start = alloc->start;
4962 for_each_plane_id_on_crtc(crtc, plane_id) {
4963 struct skl_ddb_entry *plane_alloc =
4964 &crtc_state->wm.skl.plane_ddb_y[plane_id];
4965 struct skl_ddb_entry *uv_plane_alloc =
4966 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4967
4968 if (plane_id == PLANE_CURSOR)
4969 continue;
4970
4971 /* Gen11+ uses a separate plane for UV watermarks */
4972 drm_WARN_ON(&dev_priv->drm,
4973 INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4974
4975 /* Leave disabled planes at (0,0) */
4976 if (total[plane_id]) {
4977 plane_alloc->start = start;
4978 start += total[plane_id];
4979 plane_alloc->end = start;
4980 }
4981
4982 if (uv_total[plane_id]) {
4983 uv_plane_alloc->start = start;
4984 start += uv_total[plane_id];
4985 uv_plane_alloc->end = start;
4986 }
4987 }
4988
4989 /*
4990 * When we calculated watermark values we didn't know how high
4991 * of a level we'd actually be able to hit, so we just marked
4992 * all levels as "enabled." Go back now and disable the ones
4993 * that aren't actually possible.
4994 */
4995 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4996 for_each_plane_id_on_crtc(crtc, plane_id) {
4997 struct skl_plane_wm *wm =
4998 &crtc_state->wm.skl.optimal.planes[plane_id];
4999
5000 /*
5001 * We only disable the watermarks for each plane if
5002 * they exceed the ddb allocation of said plane. This
5003 * is done so that we don't end up touching cursor
5004 * watermarks needlessly when some other plane reduces
5005 * our max possible watermark level.
5006 *
5007 * Bspec has this to say about the PLANE_WM enable bit:
5008 * "All the watermarks at this level for all enabled
5009 * planes must be enabled before the level will be used."
5010 * So this is actually safe to do.
5011 */
5012 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
5013 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
5014 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
5015
5016 /*
5017 * Wa_1408961008:icl, ehl
5018 * Underruns with WM1+ disabled
5019 */
5020 if (IS_GEN(dev_priv, 11) &&
5021 level == 1 && wm->wm[0].plane_en) {
5022 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
5023 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
5024 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5025 }
5026 }
5027 }
5028
5029 /*
5030 * Go back and disable the transition watermark if it turns out we
5031 * don't have enough DDB blocks for it.
5032 */
5033 for_each_plane_id_on_crtc(crtc, plane_id) {
5034 struct skl_plane_wm *wm =
5035 &crtc_state->wm.skl.optimal.planes[plane_id];
5036
5037 if (wm->trans_wm.plane_res_b >= total[plane_id])
5038 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
5039 }
5040
5041 return 0;
5042}
5043
5044/*
5045 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5046 * for the read latency) and cpp should always be <= 8, so that
5047 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5048 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5049*/
5050static uint_fixed_16_16_t
5051skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5052 u8 cpp, u32 latency, u32 dbuf_block_size)
5053{
5054 u32 wm_intermediate_val;
5055 uint_fixed_16_16_t ret;
5056
5057 if (latency == 0)
5058 return FP_16_16_MAX;
5059
5060 wm_intermediate_val = latency * pixel_rate * cpp;
5061 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5062
5063 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5064 ret = add_fixed16_u32(ret, 1);
5065
5066 return ret;
5067}
5068
5069static uint_fixed_16_16_t
5070skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5071 uint_fixed_16_16_t plane_blocks_per_line)
5072{
5073 u32 wm_intermediate_val;
5074 uint_fixed_16_16_t ret;
5075
5076 if (latency == 0)
5077 return FP_16_16_MAX;
5078
5079 wm_intermediate_val = latency * pixel_rate;
5080 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5081 pipe_htotal * 1000);
5082 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5083 return ret;
5084}
5085
5086static uint_fixed_16_16_t
5087intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5088{
5089 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5090 u32 pixel_rate;
5091 u32 crtc_htotal;
5092 uint_fixed_16_16_t linetime_us;
5093
5094 if (!crtc_state->hw.active)
5095 return u32_to_fixed16(0);
5096
5097 pixel_rate = crtc_state->pixel_rate;
5098
5099 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5100 return u32_to_fixed16(0);
5101
5102 crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
5103 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5104
5105 return linetime_us;
5106}
5107
5108static u32
5109skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
5110 const struct intel_plane_state *plane_state)
5111{
5112 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5113 u64 adjusted_pixel_rate;
5114 uint_fixed_16_16_t downscale_amount;
5115
5116 /* Shouldn't reach here on disabled planes... */
5117 if (drm_WARN_ON(&dev_priv->drm,
5118 !intel_wm_plane_visible(crtc_state, plane_state)))
5119 return 0;
5120
5121 /*
5122 * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
5123 * with additional adjustments for plane-specific scaling.
5124 */
5125 adjusted_pixel_rate = crtc_state->pixel_rate;
5126 downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
5127
5128 return mul_round_up_u32_fixed16(adjusted_pixel_rate,
5129 downscale_amount);
5130}
5131
5132static int
5133skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5134 int width, const struct drm_format_info *format,
5135 u64 modifier, unsigned int rotation,
5136 u32 plane_pixel_rate, struct skl_wm_params *wp,
5137 int color_plane)
5138{
5139 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5140 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5141 u32 interm_pbpl;
5142
5143 /* only planar format has two planes */
5144 if (color_plane == 1 &&
5145 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5146 drm_dbg_kms(&dev_priv->drm,
5147 "Non planar format have single plane\n");
5148 return -EINVAL;
5149 }
5150
5151 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5152 modifier == I915_FORMAT_MOD_Yf_TILED ||
5153 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5154 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5155 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5156 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5157 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5158 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5159
5160 wp->width = width;
5161 if (color_plane == 1 && wp->is_planar)
5162 wp->width /= 2;
5163
5164 wp->cpp = format->cpp[color_plane];
5165 wp->plane_pixel_rate = plane_pixel_rate;
5166
5167 if (INTEL_GEN(dev_priv) >= 11 &&
5168 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5169 wp->dbuf_block_size = 256;
5170 else
5171 wp->dbuf_block_size = 512;
5172
5173 if (drm_rotation_90_or_270(rotation)) {
5174 switch (wp->cpp) {
5175 case 1:
5176 wp->y_min_scanlines = 16;
5177 break;
5178 case 2:
5179 wp->y_min_scanlines = 8;
5180 break;
5181 case 4:
5182 wp->y_min_scanlines = 4;
5183 break;
5184 default:
5185 MISSING_CASE(wp->cpp);
5186 return -EINVAL;
5187 }
5188 } else {
5189 wp->y_min_scanlines = 4;
5190 }
5191
5192 if (skl_needs_memory_bw_wa(dev_priv))
5193 wp->y_min_scanlines *= 2;
5194
5195 wp->plane_bytes_per_line = wp->width * wp->cpp;
5196 if (wp->y_tiled) {
5197 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5198 wp->y_min_scanlines,
5199 wp->dbuf_block_size);
5200
5201 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5202 interm_pbpl++;
5203
5204 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5205 wp->y_min_scanlines);
5206 } else {
5207 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5208 wp->dbuf_block_size);
5209
5210 if (!wp->x_tiled ||
5211 INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5212 interm_pbpl++;
5213
5214 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5215 }
5216
5217 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5218 wp->plane_blocks_per_line);
5219
5220 wp->linetime_us = fixed16_to_u32_round_up(
5221 intel_get_linetime_us(crtc_state));
5222
5223 return 0;
5224}
5225
5226static int
5227skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5228 const struct intel_plane_state *plane_state,
5229 struct skl_wm_params *wp, int color_plane)
5230{
5231 const struct drm_framebuffer *fb = plane_state->hw.fb;
5232 int width;
5233
5234 /*
5235 * Src coordinates are already rotated by 270 degrees for
5236 * the 90/270 degree plane rotation cases (to match the
5237 * GTT mapping), hence no need to account for rotation here.
5238 */
5239 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5240
5241 return skl_compute_wm_params(crtc_state, width,
5242 fb->format, fb->modifier,
5243 plane_state->hw.rotation,
5244 skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
5245 wp, color_plane);
5246}
5247
5248static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5249{
5250 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5251 return true;
5252
5253 /* The number of lines are ignored for the level 0 watermark. */
5254 return level > 0;
5255}
5256
5257static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5258 int level,
5259 unsigned int latency,
5260 const struct skl_wm_params *wp,
5261 const struct skl_wm_level *result_prev,
5262 struct skl_wm_level *result /* out */)
5263{
5264 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5265 uint_fixed_16_16_t method1, method2;
5266 uint_fixed_16_16_t selected_result;
5267 u32 res_blocks, res_lines, min_ddb_alloc = 0;
5268
5269 if (latency == 0) {
5270 /* reject it */
5271 result->min_ddb_alloc = U16_MAX;
5272 return;
5273 }
5274
5275 /*
5276 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5277 * Display WA #1141: kbl,cfl
5278 */
5279 if ((IS_KABYLAKE(dev_priv) ||
5280 IS_COFFEELAKE(dev_priv) ||
5281 IS_COMETLAKE(dev_priv)) &&
5282 dev_priv->ipc_enabled)
5283 latency += 4;
5284
5285 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5286 latency += 15;
5287
5288 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5289 wp->cpp, latency, wp->dbuf_block_size);
5290 method2 = skl_wm_method2(wp->plane_pixel_rate,
5291 crtc_state->hw.adjusted_mode.crtc_htotal,
5292 latency,
5293 wp->plane_blocks_per_line);
5294
5295 if (wp->y_tiled) {
5296 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5297 } else {
5298 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5299 wp->dbuf_block_size < 1) &&
5300 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5301 selected_result = method2;
5302 } else if (latency >= wp->linetime_us) {
5303 if (IS_GEN(dev_priv, 9) &&
5304 !IS_GEMINILAKE(dev_priv))
5305 selected_result = min_fixed16(method1, method2);
5306 else
5307 selected_result = method2;
5308 } else {
5309 selected_result = method1;
5310 }
5311 }
5312
5313 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5314 res_lines = div_round_up_fixed16(selected_result,
5315 wp->plane_blocks_per_line);
5316
5317 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5318 /* Display WA #1125: skl,bxt,kbl */
5319 if (level == 0 && wp->rc_surface)
5320 res_blocks +=
5321 fixed16_to_u32_round_up(wp->y_tile_minimum);
5322
5323 /* Display WA #1126: skl,bxt,kbl */
5324 if (level >= 1 && level <= 7) {
5325 if (wp->y_tiled) {
5326 res_blocks +=
5327 fixed16_to_u32_round_up(wp->y_tile_minimum);
5328 res_lines += wp->y_min_scanlines;
5329 } else {
5330 res_blocks++;
5331 }
5332
5333 /*
5334 * Make sure result blocks for higher latency levels are
5335 * atleast as high as level below the current level.
5336 * Assumption in DDB algorithm optimization for special
5337 * cases. Also covers Display WA #1125 for RC.
5338 */
5339 if (result_prev->plane_res_b > res_blocks)
5340 res_blocks = result_prev->plane_res_b;
5341 }
5342 }
5343
5344 if (INTEL_GEN(dev_priv) >= 11) {
5345 if (wp->y_tiled) {
5346 int extra_lines;
5347
5348 if (res_lines % wp->y_min_scanlines == 0)
5349 extra_lines = wp->y_min_scanlines;
5350 else
5351 extra_lines = wp->y_min_scanlines * 2 -
5352 res_lines % wp->y_min_scanlines;
5353
5354 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5355 wp->plane_blocks_per_line);
5356 } else {
5357 min_ddb_alloc = res_blocks +
5358 DIV_ROUND_UP(res_blocks, 10);
5359 }
5360 }
5361
5362 if (!skl_wm_has_lines(dev_priv, level))
5363 res_lines = 0;
5364
5365 if (res_lines > 31) {
5366 /* reject it */
5367 result->min_ddb_alloc = U16_MAX;
5368 return;
5369 }
5370
5371 /*
5372 * If res_lines is valid, assume we can use this watermark level
5373 * for now. We'll come back and disable it after we calculate the
5374 * DDB allocation if it turns out we don't actually have enough
5375 * blocks to satisfy it.
5376 */
5377 result->plane_res_b = res_blocks;
5378 result->plane_res_l = res_lines;
5379 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5380 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5381 result->plane_en = true;
5382}
5383
5384static void
5385skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5386 const struct skl_wm_params *wm_params,
5387 struct skl_wm_level *levels)
5388{
5389 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5390 int level, max_level = ilk_wm_max_level(dev_priv);
5391 struct skl_wm_level *result_prev = &levels[0];
5392
5393 for (level = 0; level <= max_level; level++) {
5394 struct skl_wm_level *result = &levels[level];
5395 unsigned int latency = dev_priv->wm.skl_latency[level];
5396
5397 skl_compute_plane_wm(crtc_state, level, latency,
5398 wm_params, result_prev, result);
5399
5400 result_prev = result;
5401 }
5402}
5403
5404static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5405 const struct skl_wm_params *wm_params,
5406 struct skl_plane_wm *plane_wm)
5407{
5408 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5409 struct skl_wm_level *sagv_wm = &plane_wm->sagv_wm0;
5410 struct skl_wm_level *levels = plane_wm->wm;
5411 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5412
5413 skl_compute_plane_wm(crtc_state, 0, latency,
5414 wm_params, &levels[0],
5415 sagv_wm);
5416}
5417
5418static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5419 const struct skl_wm_params *wp,
5420 struct skl_plane_wm *wm)
5421{
5422 struct drm_device *dev = crtc_state->uapi.crtc->dev;
5423 const struct drm_i915_private *dev_priv = to_i915(dev);
5424 u16 trans_min, trans_amount, trans_y_tile_min;
5425 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5426
5427 /* Transition WM don't make any sense if ipc is disabled */
5428 if (!dev_priv->ipc_enabled)
5429 return;
5430
5431 /*
5432 * WaDisableTWM:skl,kbl,cfl,bxt
5433 * Transition WM are not recommended by HW team for GEN9
5434 */
5435 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5436 return;
5437
5438 if (INTEL_GEN(dev_priv) >= 11)
5439 trans_min = 4;
5440 else
5441 trans_min = 14;
5442
5443 /* Display WA #1140: glk,cnl */
5444 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5445 trans_amount = 0;
5446 else
5447 trans_amount = 10; /* This is configurable amount */
5448
5449 trans_offset_b = trans_min + trans_amount;
5450
5451 /*
5452 * The spec asks for Selected Result Blocks for wm0 (the real value),
5453 * not Result Blocks (the integer value). Pay attention to the capital
5454 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5455 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5456 * and since we later will have to get the ceiling of the sum in the
5457 * transition watermarks calculation, we can just pretend Selected
5458 * Result Blocks is Result Blocks minus 1 and it should work for the
5459 * current platforms.
5460 */
5461 wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5462
5463 if (wp->y_tiled) {
5464 trans_y_tile_min =
5465 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5466 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5467 trans_offset_b;
5468 } else {
5469 res_blocks = wm0_sel_res_b + trans_offset_b;
5470 }
5471
5472 /*
5473 * Just assume we can enable the transition watermark. After
5474 * computing the DDB we'll come back and disable it if that
5475 * assumption turns out to be false.
5476 */
5477 wm->trans_wm.plane_res_b = res_blocks + 1;
5478 wm->trans_wm.plane_en = true;
5479}
5480
5481static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5482 const struct intel_plane_state *plane_state,
5483 enum plane_id plane_id, int color_plane)
5484{
5485 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5486 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5487 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5488 struct skl_wm_params wm_params;
5489 int ret;
5490
5491 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5492 &wm_params, color_plane);
5493 if (ret)
5494 return ret;
5495
5496 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5497
5498 if (INTEL_GEN(dev_priv) >= 12)
5499 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5500
5501 skl_compute_transition_wm(crtc_state, &wm_params, wm);
5502
5503 return 0;
5504}
5505
5506static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5507 const struct intel_plane_state *plane_state,
5508 enum plane_id plane_id)
5509{
5510 struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5511 struct skl_wm_params wm_params;
5512 int ret;
5513
5514 wm->is_planar = true;
5515
5516 /* uv plane watermarks must also be validated for NV12/Planar */
5517 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5518 &wm_params, 1);
5519 if (ret)
5520 return ret;
5521
5522 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5523
5524 return 0;
5525}
5526
5527static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5528 const struct intel_plane_state *plane_state)
5529{
5530 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5531 const struct drm_framebuffer *fb = plane_state->hw.fb;
5532 enum plane_id plane_id = plane->id;
5533 int ret;
5534
5535 if (!intel_wm_plane_visible(crtc_state, plane_state))
5536 return 0;
5537
5538 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5539 plane_id, 0);
5540 if (ret)
5541 return ret;
5542
5543 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5544 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5545 plane_id);
5546 if (ret)
5547 return ret;
5548 }
5549
5550 return 0;
5551}
5552
5553static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5554 const struct intel_plane_state *plane_state)
5555{
5556 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5557 enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5558 int ret;
5559
5560 /* Watermarks calculated in master */
5561 if (plane_state->planar_slave)
5562 return 0;
5563
5564 if (plane_state->planar_linked_plane) {
5565 const struct drm_framebuffer *fb = plane_state->hw.fb;
5566 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5567
5568 drm_WARN_ON(&dev_priv->drm,
5569 !intel_wm_plane_visible(crtc_state, plane_state));
5570 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5571 fb->format->num_planes == 1);
5572
5573 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5574 y_plane_id, 0);
5575 if (ret)
5576 return ret;
5577
5578 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5579 plane_id, 1);
5580 if (ret)
5581 return ret;
5582 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5583 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5584 plane_id, 0);
5585 if (ret)
5586 return ret;
5587 }
5588
5589 return 0;
5590}
5591
5592static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5593{
5594 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5595 struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5596 struct intel_plane *plane;
5597 const struct intel_plane_state *plane_state;
5598 int ret;
5599
5600 /*
5601 * We'll only calculate watermarks for planes that are actually
5602 * enabled, so make sure all other planes are set as disabled.
5603 */
5604 memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5605
5606 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5607 crtc_state) {
5608
5609 if (INTEL_GEN(dev_priv) >= 11)
5610 ret = icl_build_plane_wm(crtc_state, plane_state);
5611 else
5612 ret = skl_build_plane_wm(crtc_state, plane_state);
5613 if (ret)
5614 return ret;
5615 }
5616
5617 return 0;
5618}
5619
5620static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5621 i915_reg_t reg,
5622 const struct skl_ddb_entry *entry)
5623{
5624 if (entry->end)
5625 intel_de_write_fw(dev_priv, reg,
5626 (entry->end - 1) << 16 | entry->start);
5627 else
5628 intel_de_write_fw(dev_priv, reg, 0);
5629}
5630
5631static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5632 i915_reg_t reg,
5633 const struct skl_wm_level *level)
5634{
5635 u32 val = 0;
5636
5637 if (level->plane_en)
5638 val |= PLANE_WM_EN;
5639 if (level->ignore_lines)
5640 val |= PLANE_WM_IGNORE_LINES;
5641 val |= level->plane_res_b;
5642 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5643
5644 intel_de_write_fw(dev_priv, reg, val);
5645}
5646
5647void skl_write_plane_wm(struct intel_plane *plane,
5648 const struct intel_crtc_state *crtc_state)
5649{
5650 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5651 int level, max_level = ilk_wm_max_level(dev_priv);
5652 enum plane_id plane_id = plane->id;
5653 enum pipe pipe = plane->pipe;
5654 const struct skl_plane_wm *wm =
5655 &crtc_state->wm.skl.optimal.planes[plane_id];
5656 const struct skl_ddb_entry *ddb_y =
5657 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5658 const struct skl_ddb_entry *ddb_uv =
5659 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5660
5661 for (level = 0; level <= max_level; level++) {
5662 const struct skl_wm_level *wm_level;
5663
5664 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5665
5666 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5667 wm_level);
5668 }
5669 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5670 &wm->trans_wm);
5671
5672 if (INTEL_GEN(dev_priv) >= 11) {
5673 skl_ddb_entry_write(dev_priv,
5674 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5675 return;
5676 }
5677
5678 if (wm->is_planar)
5679 swap(ddb_y, ddb_uv);
5680
5681 skl_ddb_entry_write(dev_priv,
5682 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5683 skl_ddb_entry_write(dev_priv,
5684 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5685}
5686
5687void skl_write_cursor_wm(struct intel_plane *plane,
5688 const struct intel_crtc_state *crtc_state)
5689{
5690 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5691 int level, max_level = ilk_wm_max_level(dev_priv);
5692 enum plane_id plane_id = plane->id;
5693 enum pipe pipe = plane->pipe;
5694 const struct skl_plane_wm *wm =
5695 &crtc_state->wm.skl.optimal.planes[plane_id];
5696 const struct skl_ddb_entry *ddb =
5697 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5698
5699 for (level = 0; level <= max_level; level++) {
5700 const struct skl_wm_level *wm_level;
5701
5702 wm_level = skl_plane_wm_level(crtc_state, plane_id, level);
5703
5704 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5705 wm_level);
5706 }
5707 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5708
5709 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5710}
5711
5712bool skl_wm_level_equals(const struct skl_wm_level *l1,
5713 const struct skl_wm_level *l2)
5714{
5715 return l1->plane_en == l2->plane_en &&
5716 l1->ignore_lines == l2->ignore_lines &&
5717 l1->plane_res_l == l2->plane_res_l &&
5718 l1->plane_res_b == l2->plane_res_b;
5719}
5720
5721static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5722 const struct skl_plane_wm *wm1,
5723 const struct skl_plane_wm *wm2)
5724{
5725 int level, max_level = ilk_wm_max_level(dev_priv);
5726
5727 for (level = 0; level <= max_level; level++) {
5728 /*
5729 * We don't check uv_wm as the hardware doesn't actually
5730 * use it. It only gets used for calculating the required
5731 * ddb allocation.
5732 */
5733 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5734 return false;
5735 }
5736
5737 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5738}
5739
5740static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5741 const struct skl_ddb_entry *b)
5742{
5743 return a->start < b->end && b->start < a->end;
5744}
5745
5746bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5747 const struct skl_ddb_entry *entries,
5748 int num_entries, int ignore_idx)
5749{
5750 int i;
5751
5752 for (i = 0; i < num_entries; i++) {
5753 if (i != ignore_idx &&
5754 skl_ddb_entries_overlap(ddb, &entries[i]))
5755 return true;
5756 }
5757
5758 return false;
5759}
5760
5761static int
5762skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5763 struct intel_crtc_state *new_crtc_state)
5764{
5765 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5766 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5767 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5768 struct intel_plane *plane;
5769
5770 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5771 struct intel_plane_state *plane_state;
5772 enum plane_id plane_id = plane->id;
5773
5774 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5775 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5776 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5777 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5778 continue;
5779
5780 plane_state = intel_atomic_get_plane_state(state, plane);
5781 if (IS_ERR(plane_state))
5782 return PTR_ERR(plane_state);
5783
5784 new_crtc_state->update_planes |= BIT(plane_id);
5785 }
5786
5787 return 0;
5788}
5789
5790static int
5791skl_compute_ddb(struct intel_atomic_state *state)
5792{
5793 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5794 const struct intel_dbuf_state *old_dbuf_state;
5795 const struct intel_dbuf_state *new_dbuf_state;
5796 const struct intel_crtc_state *old_crtc_state;
5797 struct intel_crtc_state *new_crtc_state;
5798 struct intel_crtc *crtc;
5799 int ret, i;
5800
5801 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5802 new_crtc_state, i) {
5803 ret = skl_allocate_pipe_ddb(new_crtc_state);
5804 if (ret)
5805 return ret;
5806
5807 ret = skl_ddb_add_affected_planes(old_crtc_state,
5808 new_crtc_state);
5809 if (ret)
5810 return ret;
5811 }
5812
5813 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5814 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
5815
5816 if (new_dbuf_state &&
5817 new_dbuf_state->enabled_slices != old_dbuf_state->enabled_slices)
5818 drm_dbg_kms(&dev_priv->drm,
5819 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5820 old_dbuf_state->enabled_slices,
5821 new_dbuf_state->enabled_slices,
5822 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5823
5824 return 0;
5825}
5826
5827static char enast(bool enable)
5828{
5829 return enable ? '*' : ' ';
5830}
5831
5832static void
5833skl_print_wm_changes(struct intel_atomic_state *state)
5834{
5835 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5836 const struct intel_crtc_state *old_crtc_state;
5837 const struct intel_crtc_state *new_crtc_state;
5838 struct intel_plane *plane;
5839 struct intel_crtc *crtc;
5840 int i;
5841
5842 if (!drm_debug_enabled(DRM_UT_KMS))
5843 return;
5844
5845 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5846 new_crtc_state, i) {
5847 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5848
5849 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5850 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5851
5852 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5853 enum plane_id plane_id = plane->id;
5854 const struct skl_ddb_entry *old, *new;
5855
5856 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5857 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5858
5859 if (skl_ddb_entry_equal(old, new))
5860 continue;
5861
5862 drm_dbg_kms(&dev_priv->drm,
5863 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5864 plane->base.base.id, plane->base.name,
5865 old->start, old->end, new->start, new->end,
5866 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5867 }
5868
5869 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5870 enum plane_id plane_id = plane->id;
5871 const struct skl_plane_wm *old_wm, *new_wm;
5872
5873 old_wm = &old_pipe_wm->planes[plane_id];
5874 new_wm = &new_pipe_wm->planes[plane_id];
5875
5876 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5877 continue;
5878
5879 drm_dbg_kms(&dev_priv->drm,
5880 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm"
5881 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm\n",
5882 plane->base.base.id, plane->base.name,
5883 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5884 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5885 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5886 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5887 enast(old_wm->trans_wm.plane_en),
5888 enast(old_wm->sagv_wm0.plane_en),
5889 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5890 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5891 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5892 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5893 enast(new_wm->trans_wm.plane_en),
5894 enast(new_wm->sagv_wm0.plane_en));
5895
5896 drm_dbg_kms(&dev_priv->drm,
5897 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5898 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5899 plane->base.base.id, plane->base.name,
5900 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5901 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5902 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5903 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5904 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5905 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5906 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5907 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5908 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5909 enast(old_wm->sagv_wm0.ignore_lines), old_wm->sagv_wm0.plane_res_l,
5910
5911 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5912 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5913 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5914 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5915 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5916 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5917 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5918 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5919 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
5920 enast(new_wm->sagv_wm0.ignore_lines), new_wm->sagv_wm0.plane_res_l);
5921
5922 drm_dbg_kms(&dev_priv->drm,
5923 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5924 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5925 plane->base.base.id, plane->base.name,
5926 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5927 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5928 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5929 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5930 old_wm->trans_wm.plane_res_b,
5931 old_wm->sagv_wm0.plane_res_b,
5932 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5933 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5934 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5935 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5936 new_wm->trans_wm.plane_res_b,
5937 new_wm->sagv_wm0.plane_res_b);
5938
5939 drm_dbg_kms(&dev_priv->drm,
5940 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5941 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5942 plane->base.base.id, plane->base.name,
5943 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5944 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5945 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5946 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5947 old_wm->trans_wm.min_ddb_alloc,
5948 old_wm->sagv_wm0.min_ddb_alloc,
5949 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5950 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5951 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5952 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5953 new_wm->trans_wm.min_ddb_alloc,
5954 new_wm->sagv_wm0.min_ddb_alloc);
5955 }
5956 }
5957}
5958
5959static int intel_add_affected_pipes(struct intel_atomic_state *state,
5960 u8 pipe_mask)
5961{
5962 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5963 struct intel_crtc *crtc;
5964
5965 for_each_intel_crtc(&dev_priv->drm, crtc) {
5966 struct intel_crtc_state *crtc_state;
5967
5968 if ((pipe_mask & BIT(crtc->pipe)) == 0)
5969 continue;
5970
5971 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5972 if (IS_ERR(crtc_state))
5973 return PTR_ERR(crtc_state);
5974 }
5975
5976 return 0;
5977}
5978
5979static int
5980skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5981{
5982 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5983 struct intel_crtc_state *crtc_state;
5984 struct intel_crtc *crtc;
5985 int i, ret;
5986
5987 if (dev_priv->wm.distrust_bios_wm) {
5988 /*
5989 * skl_ddb_get_pipe_allocation_limits() currently requires
5990 * all active pipes to be included in the state so that
5991 * it can redistribute the dbuf among them, and it really
5992 * wants to recompute things when distrust_bios_wm is set
5993 * so we add all the pipes to the state.
5994 */
5995 ret = intel_add_affected_pipes(state, ~0);
5996 if (ret)
5997 return ret;
5998 }
5999
6000 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6001 struct intel_dbuf_state *new_dbuf_state;
6002 const struct intel_dbuf_state *old_dbuf_state;
6003
6004 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6005 if (IS_ERR(new_dbuf_state))
6006 return PTR_ERR(new_dbuf_state);
6007
6008 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6009
6010 new_dbuf_state->active_pipes =
6011 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6012
6013 if (old_dbuf_state->active_pipes == new_dbuf_state->active_pipes)
6014 break;
6015
6016 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6017 if (ret)
6018 return ret;
6019
6020 /*
6021 * skl_ddb_get_pipe_allocation_limits() currently requires
6022 * all active pipes to be included in the state so that
6023 * it can redistribute the dbuf among them.
6024 */
6025 ret = intel_add_affected_pipes(state,
6026 new_dbuf_state->active_pipes);
6027 if (ret)
6028 return ret;
6029
6030 break;
6031 }
6032
6033 return 0;
6034}
6035
6036/*
6037 * To make sure the cursor watermark registers are always consistent
6038 * with our computed state the following scenario needs special
6039 * treatment:
6040 *
6041 * 1. enable cursor
6042 * 2. move cursor entirely offscreen
6043 * 3. disable cursor
6044 *
6045 * Step 2. does call .disable_plane() but does not zero the watermarks
6046 * (since we consider an offscreen cursor still active for the purposes
6047 * of watermarks). Step 3. would not normally call .disable_plane()
6048 * because the actual plane visibility isn't changing, and we don't
6049 * deallocate the cursor ddb until the pipe gets disabled. So we must
6050 * force step 3. to call .disable_plane() to update the watermark
6051 * registers properly.
6052 *
6053 * Other planes do not suffer from this issues as their watermarks are
6054 * calculated based on the actual plane visibility. The only time this
6055 * can trigger for the other planes is during the initial readout as the
6056 * default value of the watermarks registers is not zero.
6057 */
6058static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6059 struct intel_crtc *crtc)
6060{
6061 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6062 const struct intel_crtc_state *old_crtc_state =
6063 intel_atomic_get_old_crtc_state(state, crtc);
6064 struct intel_crtc_state *new_crtc_state =
6065 intel_atomic_get_new_crtc_state(state, crtc);
6066 struct intel_plane *plane;
6067
6068 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6069 struct intel_plane_state *plane_state;
6070 enum plane_id plane_id = plane->id;
6071
6072 /*
6073 * Force a full wm update for every plane on modeset.
6074 * Required because the reset value of the wm registers
6075 * is non-zero, whereas we want all disabled planes to
6076 * have zero watermarks. So if we turn off the relevant
6077 * power well the hardware state will go out of sync
6078 * with the software state.
6079 */
6080 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6081 skl_plane_wm_equals(dev_priv,
6082 &old_crtc_state->wm.skl.optimal.planes[plane_id],
6083 &new_crtc_state->wm.skl.optimal.planes[plane_id]))
6084 continue;
6085
6086 plane_state = intel_atomic_get_plane_state(state, plane);
6087 if (IS_ERR(plane_state))
6088 return PTR_ERR(plane_state);
6089
6090 new_crtc_state->update_planes |= BIT(plane_id);
6091 }
6092
6093 return 0;
6094}
6095
6096static int
6097skl_compute_wm(struct intel_atomic_state *state)
6098{
6099 struct intel_crtc *crtc;
6100 struct intel_crtc_state *new_crtc_state;
6101 struct intel_crtc_state *old_crtc_state;
6102 int ret, i;
6103
6104 ret = skl_ddb_add_affected_pipes(state);
6105 if (ret)
6106 return ret;
6107
6108 /*
6109 * Calculate WM's for all pipes that are part of this transaction.
6110 * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
6111 * weren't otherwise being modified if pipe allocations had to change.
6112 */
6113 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6114 new_crtc_state, i) {
6115 ret = skl_build_pipe_wm(new_crtc_state);
6116 if (ret)
6117 return ret;
6118 }
6119
6120 ret = skl_compute_ddb(state);
6121 if (ret)
6122 return ret;
6123
6124 ret = intel_compute_sagv_mask(state);
6125 if (ret)
6126 return ret;
6127
6128 /*
6129 * skl_compute_ddb() will have adjusted the final watermarks
6130 * based on how much ddb is available. Now we can actually
6131 * check if the final watermarks changed.
6132 */
6133 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6134 new_crtc_state, i) {
6135 ret = skl_wm_add_affected_planes(state, crtc);
6136 if (ret)
6137 return ret;
6138 }
6139
6140 skl_print_wm_changes(state);
6141
6142 return 0;
6143}
6144
6145static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6146 struct intel_wm_config *config)
6147{
6148 struct intel_crtc *crtc;
6149
6150 /* Compute the currently _active_ config */
6151 for_each_intel_crtc(&dev_priv->drm, crtc) {
6152 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6153
6154 if (!wm->pipe_enabled)
6155 continue;
6156
6157 config->sprites_enabled |= wm->sprites_enabled;
6158 config->sprites_scaled |= wm->sprites_scaled;
6159 config->num_pipes_active++;
6160 }
6161}
6162
6163static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6164{
6165 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6166 struct ilk_wm_maximums max;
6167 struct intel_wm_config config = {};
6168 struct ilk_wm_values results = {};
6169 enum intel_ddb_partitioning partitioning;
6170
6171 ilk_compute_wm_config(dev_priv, &config);
6172
6173 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6174 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6175
6176 /* 5/6 split only in single pipe config on IVB+ */
6177 if (INTEL_GEN(dev_priv) >= 7 &&
6178 config.num_pipes_active == 1 && config.sprites_enabled) {
6179 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6180 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6181
6182 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6183 } else {
6184 best_lp_wm = &lp_wm_1_2;
6185 }
6186
6187 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6188 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6189
6190 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6191
6192 ilk_write_wm_values(dev_priv, &results);
6193}
6194
6195static void ilk_initial_watermarks(struct intel_atomic_state *state,
6196 struct intel_crtc *crtc)
6197{
6198 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6199 const struct intel_crtc_state *crtc_state =
6200 intel_atomic_get_new_crtc_state(state, crtc);
6201
6202 mutex_lock(&dev_priv->wm.wm_mutex);
6203 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6204 ilk_program_watermarks(dev_priv);
6205 mutex_unlock(&dev_priv->wm.wm_mutex);
6206}
6207
6208static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6209 struct intel_crtc *crtc)
6210{
6211 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6212 const struct intel_crtc_state *crtc_state =
6213 intel_atomic_get_new_crtc_state(state, crtc);
6214
6215 if (!crtc_state->wm.need_postvbl_update)
6216 return;
6217
6218 mutex_lock(&dev_priv->wm.wm_mutex);
6219 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6220 ilk_program_watermarks(dev_priv);
6221 mutex_unlock(&dev_priv->wm.wm_mutex);
6222}
6223
6224static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6225{
6226 level->plane_en = val & PLANE_WM_EN;
6227 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6228 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6229 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6230 PLANE_WM_LINES_MASK;
6231}
6232
6233void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6234 struct skl_pipe_wm *out)
6235{
6236 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6237 enum pipe pipe = crtc->pipe;
6238 int level, max_level;
6239 enum plane_id plane_id;
6240 u32 val;
6241
6242 max_level = ilk_wm_max_level(dev_priv);
6243
6244 for_each_plane_id_on_crtc(crtc, plane_id) {
6245 struct skl_plane_wm *wm = &out->planes[plane_id];
6246
6247 for (level = 0; level <= max_level; level++) {
6248 if (plane_id != PLANE_CURSOR)
6249 val = I915_READ(PLANE_WM(pipe, plane_id, level));
6250 else
6251 val = I915_READ(CUR_WM(pipe, level));
6252
6253 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6254 }
6255
6256 if (INTEL_GEN(dev_priv) >= 12)
6257 wm->sagv_wm0 = wm->wm[0];
6258
6259 if (plane_id != PLANE_CURSOR)
6260 val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
6261 else
6262 val = I915_READ(CUR_WM_TRANS(pipe));
6263
6264 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6265 }
6266
6267 if (!crtc->active)
6268 return;
6269}
6270
6271void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6272{
6273 struct intel_crtc *crtc;
6274 struct intel_crtc_state *crtc_state;
6275
6276 for_each_intel_crtc(&dev_priv->drm, crtc) {
6277 crtc_state = to_intel_crtc_state(crtc->base.state);
6278
6279 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6280 }
6281
6282 if (dev_priv->active_pipes) {
6283 /* Fully recompute DDB on first atomic commit */
6284 dev_priv->wm.distrust_bios_wm = true;
6285 }
6286}
6287
6288static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6289{
6290 struct drm_device *dev = crtc->base.dev;
6291 struct drm_i915_private *dev_priv = to_i915(dev);
6292 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6293 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6294 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6295 enum pipe pipe = crtc->pipe;
6296 static const i915_reg_t wm0_pipe_reg[] = {
6297 [PIPE_A] = WM0_PIPEA_ILK,
6298 [PIPE_B] = WM0_PIPEB_ILK,
6299 [PIPE_C] = WM0_PIPEC_IVB,
6300 };
6301
6302 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
6303
6304 memset(active, 0, sizeof(*active));
6305
6306 active->pipe_enabled = crtc->active;
6307
6308 if (active->pipe_enabled) {
6309 u32 tmp = hw->wm_pipe[pipe];
6310
6311 /*
6312 * For active pipes LP0 watermark is marked as
6313 * enabled, and LP1+ watermaks as disabled since
6314 * we can't really reverse compute them in case
6315 * multiple pipes are active.
6316 */
6317 active->wm[0].enable = true;
6318 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6319 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6320 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6321 } else {
6322 int level, max_level = ilk_wm_max_level(dev_priv);
6323
6324 /*
6325 * For inactive pipes, all watermark levels
6326 * should be marked as enabled but zeroed,
6327 * which is what we'd compute them to.
6328 */
6329 for (level = 0; level <= max_level; level++)
6330 active->wm[level].enable = true;
6331 }
6332
6333 crtc->wm.active.ilk = *active;
6334}
6335
6336#define _FW_WM(value, plane) \
6337 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6338#define _FW_WM_VLV(value, plane) \
6339 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6340
6341static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6342 struct g4x_wm_values *wm)
6343{
6344 u32 tmp;
6345
6346 tmp = I915_READ(DSPFW1);
6347 wm->sr.plane = _FW_WM(tmp, SR);
6348 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6349 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6350 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6351
6352 tmp = I915_READ(DSPFW2);
6353 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6354 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6355 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6356 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6357 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6358 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6359
6360 tmp = I915_READ(DSPFW3);
6361 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6362 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6363 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6364 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6365}
6366
6367static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6368 struct vlv_wm_values *wm)
6369{
6370 enum pipe pipe;
6371 u32 tmp;
6372
6373 for_each_pipe(dev_priv, pipe) {
6374 tmp = I915_READ(VLV_DDL(pipe));
6375
6376 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6377 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6378 wm->ddl[pipe].plane[PLANE_CURSOR] =
6379 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6380 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6381 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6382 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6383 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6384 }
6385
6386 tmp = I915_READ(DSPFW1);
6387 wm->sr.plane = _FW_WM(tmp, SR);
6388 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6389 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6390 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6391
6392 tmp = I915_READ(DSPFW2);
6393 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6394 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6395 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6396
6397 tmp = I915_READ(DSPFW3);
6398 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6399
6400 if (IS_CHERRYVIEW(dev_priv)) {
6401 tmp = I915_READ(DSPFW7_CHV);
6402 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6403 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6404
6405 tmp = I915_READ(DSPFW8_CHV);
6406 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6407 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6408
6409 tmp = I915_READ(DSPFW9_CHV);
6410 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6411 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6412
6413 tmp = I915_READ(DSPHOWM);
6414 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6415 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6416 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6417 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6418 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6419 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6420 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6421 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6422 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6423 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6424 } else {
6425 tmp = I915_READ(DSPFW7);
6426 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6427 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6428
6429 tmp = I915_READ(DSPHOWM);
6430 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6431 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6432 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6433 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6434 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6435 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6436 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6437 }
6438}
6439
6440#undef _FW_WM
6441#undef _FW_WM_VLV
6442
6443void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6444{
6445 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6446 struct intel_crtc *crtc;
6447
6448 g4x_read_wm_values(dev_priv, wm);
6449
6450 wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6451
6452 for_each_intel_crtc(&dev_priv->drm, crtc) {
6453 struct intel_crtc_state *crtc_state =
6454 to_intel_crtc_state(crtc->base.state);
6455 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6456 struct g4x_pipe_wm *raw;
6457 enum pipe pipe = crtc->pipe;
6458 enum plane_id plane_id;
6459 int level, max_level;
6460
6461 active->cxsr = wm->cxsr;
6462 active->hpll_en = wm->hpll_en;
6463 active->fbc_en = wm->fbc_en;
6464
6465 active->sr = wm->sr;
6466 active->hpll = wm->hpll;
6467
6468 for_each_plane_id_on_crtc(crtc, plane_id) {
6469 active->wm.plane[plane_id] =
6470 wm->pipe[pipe].plane[plane_id];
6471 }
6472
6473 if (wm->cxsr && wm->hpll_en)
6474 max_level = G4X_WM_LEVEL_HPLL;
6475 else if (wm->cxsr)
6476 max_level = G4X_WM_LEVEL_SR;
6477 else
6478 max_level = G4X_WM_LEVEL_NORMAL;
6479
6480 level = G4X_WM_LEVEL_NORMAL;
6481 raw = &crtc_state->wm.g4x.raw[level];
6482 for_each_plane_id_on_crtc(crtc, plane_id)
6483 raw->plane[plane_id] = active->wm.plane[plane_id];
6484
6485 if (++level > max_level)
6486 goto out;
6487
6488 raw = &crtc_state->wm.g4x.raw[level];
6489 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6490 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6491 raw->plane[PLANE_SPRITE0] = 0;
6492 raw->fbc = active->sr.fbc;
6493
6494 if (++level > max_level)
6495 goto out;
6496
6497 raw = &crtc_state->wm.g4x.raw[level];
6498 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6499 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6500 raw->plane[PLANE_SPRITE0] = 0;
6501 raw->fbc = active->hpll.fbc;
6502
6503 out:
6504 for_each_plane_id_on_crtc(crtc, plane_id)
6505 g4x_raw_plane_wm_set(crtc_state, level,
6506 plane_id, USHRT_MAX);
6507 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6508
6509 crtc_state->wm.g4x.optimal = *active;
6510 crtc_state->wm.g4x.intermediate = *active;
6511
6512 drm_dbg_kms(&dev_priv->drm,
6513 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6514 pipe_name(pipe),
6515 wm->pipe[pipe].plane[PLANE_PRIMARY],
6516 wm->pipe[pipe].plane[PLANE_CURSOR],
6517 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6518 }
6519
6520 drm_dbg_kms(&dev_priv->drm,
6521 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6522 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6523 drm_dbg_kms(&dev_priv->drm,
6524 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6525 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6526 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6527 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6528}
6529
6530void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6531{
6532 struct intel_plane *plane;
6533 struct intel_crtc *crtc;
6534
6535 mutex_lock(&dev_priv->wm.wm_mutex);
6536
6537 for_each_intel_plane(&dev_priv->drm, plane) {
6538 struct intel_crtc *crtc =
6539 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6540 struct intel_crtc_state *crtc_state =
6541 to_intel_crtc_state(crtc->base.state);
6542 struct intel_plane_state *plane_state =
6543 to_intel_plane_state(plane->base.state);
6544 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6545 enum plane_id plane_id = plane->id;
6546 int level;
6547
6548 if (plane_state->uapi.visible)
6549 continue;
6550
6551 for (level = 0; level < 3; level++) {
6552 struct g4x_pipe_wm *raw =
6553 &crtc_state->wm.g4x.raw[level];
6554
6555 raw->plane[plane_id] = 0;
6556 wm_state->wm.plane[plane_id] = 0;
6557 }
6558
6559 if (plane_id == PLANE_PRIMARY) {
6560 for (level = 0; level < 3; level++) {
6561 struct g4x_pipe_wm *raw =
6562 &crtc_state->wm.g4x.raw[level];
6563 raw->fbc = 0;
6564 }
6565
6566 wm_state->sr.fbc = 0;
6567 wm_state->hpll.fbc = 0;
6568 wm_state->fbc_en = false;
6569 }
6570 }
6571
6572 for_each_intel_crtc(&dev_priv->drm, crtc) {
6573 struct intel_crtc_state *crtc_state =
6574 to_intel_crtc_state(crtc->base.state);
6575
6576 crtc_state->wm.g4x.intermediate =
6577 crtc_state->wm.g4x.optimal;
6578 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6579 }
6580
6581 g4x_program_watermarks(dev_priv);
6582
6583 mutex_unlock(&dev_priv->wm.wm_mutex);
6584}
6585
6586void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6587{
6588 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6589 struct intel_crtc *crtc;
6590 u32 val;
6591
6592 vlv_read_wm_values(dev_priv, wm);
6593
6594 wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6595 wm->level = VLV_WM_LEVEL_PM2;
6596
6597 if (IS_CHERRYVIEW(dev_priv)) {
6598 vlv_punit_get(dev_priv);
6599
6600 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6601 if (val & DSP_MAXFIFO_PM5_ENABLE)
6602 wm->level = VLV_WM_LEVEL_PM5;
6603
6604 /*
6605 * If DDR DVFS is disabled in the BIOS, Punit
6606 * will never ack the request. So if that happens
6607 * assume we don't have to enable/disable DDR DVFS
6608 * dynamically. To test that just set the REQ_ACK
6609 * bit to poke the Punit, but don't change the
6610 * HIGH/LOW bits so that we don't actually change
6611 * the current state.
6612 */
6613 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6614 val |= FORCE_DDR_FREQ_REQ_ACK;
6615 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6616
6617 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6618 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6619 drm_dbg_kms(&dev_priv->drm,
6620 "Punit not acking DDR DVFS request, "
6621 "assuming DDR DVFS is disabled\n");
6622 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6623 } else {
6624 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6625 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6626 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6627 }
6628
6629 vlv_punit_put(dev_priv);
6630 }
6631
6632 for_each_intel_crtc(&dev_priv->drm, crtc) {
6633 struct intel_crtc_state *crtc_state =
6634 to_intel_crtc_state(crtc->base.state);
6635 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6636 const struct vlv_fifo_state *fifo_state =
6637 &crtc_state->wm.vlv.fifo_state;
6638 enum pipe pipe = crtc->pipe;
6639 enum plane_id plane_id;
6640 int level;
6641
6642 vlv_get_fifo_size(crtc_state);
6643
6644 active->num_levels = wm->level + 1;
6645 active->cxsr = wm->cxsr;
6646
6647 for (level = 0; level < active->num_levels; level++) {
6648 struct g4x_pipe_wm *raw =
6649 &crtc_state->wm.vlv.raw[level];
6650
6651 active->sr[level].plane = wm->sr.plane;
6652 active->sr[level].cursor = wm->sr.cursor;
6653
6654 for_each_plane_id_on_crtc(crtc, plane_id) {
6655 active->wm[level].plane[plane_id] =
6656 wm->pipe[pipe].plane[plane_id];
6657
6658 raw->plane[plane_id] =
6659 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6660 fifo_state->plane[plane_id]);
6661 }
6662 }
6663
6664 for_each_plane_id_on_crtc(crtc, plane_id)
6665 vlv_raw_plane_wm_set(crtc_state, level,
6666 plane_id, USHRT_MAX);
6667 vlv_invalidate_wms(crtc, active, level);
6668
6669 crtc_state->wm.vlv.optimal = *active;
6670 crtc_state->wm.vlv.intermediate = *active;
6671
6672 drm_dbg_kms(&dev_priv->drm,
6673 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6674 pipe_name(pipe),
6675 wm->pipe[pipe].plane[PLANE_PRIMARY],
6676 wm->pipe[pipe].plane[PLANE_CURSOR],
6677 wm->pipe[pipe].plane[PLANE_SPRITE0],
6678 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6679 }
6680
6681 drm_dbg_kms(&dev_priv->drm,
6682 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6683 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6684}
6685
6686void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6687{
6688 struct intel_plane *plane;
6689 struct intel_crtc *crtc;
6690
6691 mutex_lock(&dev_priv->wm.wm_mutex);
6692
6693 for_each_intel_plane(&dev_priv->drm, plane) {
6694 struct intel_crtc *crtc =
6695 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6696 struct intel_crtc_state *crtc_state =
6697 to_intel_crtc_state(crtc->base.state);
6698 struct intel_plane_state *plane_state =
6699 to_intel_plane_state(plane->base.state);
6700 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6701 const struct vlv_fifo_state *fifo_state =
6702 &crtc_state->wm.vlv.fifo_state;
6703 enum plane_id plane_id = plane->id;
6704 int level;
6705
6706 if (plane_state->uapi.visible)
6707 continue;
6708
6709 for (level = 0; level < wm_state->num_levels; level++) {
6710 struct g4x_pipe_wm *raw =
6711 &crtc_state->wm.vlv.raw[level];
6712
6713 raw->plane[plane_id] = 0;
6714
6715 wm_state->wm[level].plane[plane_id] =
6716 vlv_invert_wm_value(raw->plane[plane_id],
6717 fifo_state->plane[plane_id]);
6718 }
6719 }
6720
6721 for_each_intel_crtc(&dev_priv->drm, crtc) {
6722 struct intel_crtc_state *crtc_state =
6723 to_intel_crtc_state(crtc->base.state);
6724
6725 crtc_state->wm.vlv.intermediate =
6726 crtc_state->wm.vlv.optimal;
6727 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6728 }
6729
6730 vlv_program_watermarks(dev_priv);
6731
6732 mutex_unlock(&dev_priv->wm.wm_mutex);
6733}
6734
6735/*
6736 * FIXME should probably kill this and improve
6737 * the real watermark readout/sanitation instead
6738 */
6739static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6740{
6741 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6742 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6743 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6744
6745 /*
6746 * Don't touch WM1S_LP_EN here.
6747 * Doing so could cause underruns.
6748 */
6749}
6750
6751void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6752{
6753 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6754 struct intel_crtc *crtc;
6755
6756 ilk_init_lp_watermarks(dev_priv);
6757
6758 for_each_intel_crtc(&dev_priv->drm, crtc)
6759 ilk_pipe_wm_get_hw_state(crtc);
6760
6761 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6762 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6763 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6764
6765 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6766 if (INTEL_GEN(dev_priv) >= 7) {
6767 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6768 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6769 }
6770
6771 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6772 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6773 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6774 else if (IS_IVYBRIDGE(dev_priv))
6775 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6776 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6777
6778 hw->enable_fbc_wm =
6779 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6780}
6781
6782/**
6783 * intel_update_watermarks - update FIFO watermark values based on current modes
6784 * @crtc: the #intel_crtc on which to compute the WM
6785 *
6786 * Calculate watermark values for the various WM regs based on current mode
6787 * and plane configuration.
6788 *
6789 * There are several cases to deal with here:
6790 * - normal (i.e. non-self-refresh)
6791 * - self-refresh (SR) mode
6792 * - lines are large relative to FIFO size (buffer can hold up to 2)
6793 * - lines are small relative to FIFO size (buffer can hold more than 2
6794 * lines), so need to account for TLB latency
6795 *
6796 * The normal calculation is:
6797 * watermark = dotclock * bytes per pixel * latency
6798 * where latency is platform & configuration dependent (we assume pessimal
6799 * values here).
6800 *
6801 * The SR calculation is:
6802 * watermark = (trunc(latency/line time)+1) * surface width *
6803 * bytes per pixel
6804 * where
6805 * line time = htotal / dotclock
6806 * surface width = hdisplay for normal plane and 64 for cursor
6807 * and latency is assumed to be high, as above.
6808 *
6809 * The final value programmed to the register should always be rounded up,
6810 * and include an extra 2 entries to account for clock crossings.
6811 *
6812 * We don't use the sprite, so we can ignore that. And on Crestline we have
6813 * to set the non-SR watermarks to 8.
6814 */
6815void intel_update_watermarks(struct intel_crtc *crtc)
6816{
6817 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6818
6819 if (dev_priv->display.update_wm)
6820 dev_priv->display.update_wm(crtc);
6821}
6822
6823void intel_enable_ipc(struct drm_i915_private *dev_priv)
6824{
6825 u32 val;
6826
6827 if (!HAS_IPC(dev_priv))
6828 return;
6829
6830 val = I915_READ(DISP_ARB_CTL2);
6831
6832 if (dev_priv->ipc_enabled)
6833 val |= DISP_IPC_ENABLE;
6834 else
6835 val &= ~DISP_IPC_ENABLE;
6836
6837 I915_WRITE(DISP_ARB_CTL2, val);
6838}
6839
6840static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6841{
6842 /* Display WA #0477 WaDisableIPC: skl */
6843 if (IS_SKYLAKE(dev_priv))
6844 return false;
6845
6846 /* Display WA #1141: SKL:all KBL:all CFL */
6847 if (IS_KABYLAKE(dev_priv) ||
6848 IS_COFFEELAKE(dev_priv) ||
6849 IS_COMETLAKE(dev_priv))
6850 return dev_priv->dram_info.symmetric_memory;
6851
6852 return true;
6853}
6854
6855void intel_init_ipc(struct drm_i915_private *dev_priv)
6856{
6857 if (!HAS_IPC(dev_priv))
6858 return;
6859
6860 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6861
6862 intel_enable_ipc(dev_priv);
6863}
6864
6865static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6866{
6867 /*
6868 * On Ibex Peak and Cougar Point, we need to disable clock
6869 * gating for the panel power sequencer or it will fail to
6870 * start up when no ports are active.
6871 */
6872 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6873}
6874
6875static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6876{
6877 enum pipe pipe;
6878
6879 for_each_pipe(dev_priv, pipe) {
6880 I915_WRITE(DSPCNTR(pipe),
6881 I915_READ(DSPCNTR(pipe)) |
6882 DISPPLANE_TRICKLE_FEED_DISABLE);
6883
6884 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6885 POSTING_READ(DSPSURF(pipe));
6886 }
6887}
6888
6889static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6890{
6891 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6892
6893 /*
6894 * Required for FBC
6895 * WaFbcDisableDpfcClockGating:ilk
6896 */
6897 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6898 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6899 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6900
6901 I915_WRITE(PCH_3DCGDIS0,
6902 MARIUNIT_CLOCK_GATE_DISABLE |
6903 SVSMUNIT_CLOCK_GATE_DISABLE);
6904 I915_WRITE(PCH_3DCGDIS1,
6905 VFMUNIT_CLOCK_GATE_DISABLE);
6906
6907 /*
6908 * According to the spec the following bits should be set in
6909 * order to enable memory self-refresh
6910 * The bit 22/21 of 0x42004
6911 * The bit 5 of 0x42020
6912 * The bit 15 of 0x45000
6913 */
6914 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6915 (I915_READ(ILK_DISPLAY_CHICKEN2) |
6916 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6917 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6918 I915_WRITE(DISP_ARB_CTL,
6919 (I915_READ(DISP_ARB_CTL) |
6920 DISP_FBC_WM_DIS));
6921
6922 /*
6923 * Based on the document from hardware guys the following bits
6924 * should be set unconditionally in order to enable FBC.
6925 * The bit 22 of 0x42000
6926 * The bit 22 of 0x42004
6927 * The bit 7,8,9 of 0x42020.
6928 */
6929 if (IS_IRONLAKE_M(dev_priv)) {
6930 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6931 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6932 I915_READ(ILK_DISPLAY_CHICKEN1) |
6933 ILK_FBCQ_DIS);
6934 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6935 I915_READ(ILK_DISPLAY_CHICKEN2) |
6936 ILK_DPARB_GATE);
6937 }
6938
6939 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6940
6941 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6942 I915_READ(ILK_DISPLAY_CHICKEN2) |
6943 ILK_ELPIN_409_SELECT);
6944
6945 g4x_disable_trickle_feed(dev_priv);
6946
6947 ibx_init_clock_gating(dev_priv);
6948}
6949
6950static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6951{
6952 enum pipe pipe;
6953 u32 val;
6954
6955 /*
6956 * On Ibex Peak and Cougar Point, we need to disable clock
6957 * gating for the panel power sequencer or it will fail to
6958 * start up when no ports are active.
6959 */
6960 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6961 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6962 PCH_CPUNIT_CLOCK_GATE_DISABLE);
6963 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6964 DPLS_EDP_PPS_FIX_DIS);
6965 /* The below fixes the weird display corruption, a few pixels shifted
6966 * downward, on (only) LVDS of some HP laptops with IVY.
6967 */
6968 for_each_pipe(dev_priv, pipe) {
6969 val = I915_READ(TRANS_CHICKEN2(pipe));
6970 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6971 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6972 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6973 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6974 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6975 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6976 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6977 }
6978 /* WADP0ClockGatingDisable */
6979 for_each_pipe(dev_priv, pipe) {
6980 I915_WRITE(TRANS_CHICKEN1(pipe),
6981 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6982 }
6983}
6984
6985static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6986{
6987 u32 tmp;
6988
6989 tmp = I915_READ(MCH_SSKPD);
6990 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6991 drm_dbg_kms(&dev_priv->drm,
6992 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6993 tmp);
6994}
6995
6996static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6997{
6998 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6999
7000 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
7001
7002 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7003 I915_READ(ILK_DISPLAY_CHICKEN2) |
7004 ILK_ELPIN_409_SELECT);
7005
7006 I915_WRITE(GEN6_UCGCTL1,
7007 I915_READ(GEN6_UCGCTL1) |
7008 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7009 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7010
7011 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7012 * gating disable must be set. Failure to set it results in
7013 * flickering pixels due to Z write ordering failures after
7014 * some amount of runtime in the Mesa "fire" demo, and Unigine
7015 * Sanctuary and Tropics, and apparently anything else with
7016 * alpha test or pixel discard.
7017 *
7018 * According to the spec, bit 11 (RCCUNIT) must also be set,
7019 * but we didn't debug actual testcases to find it out.
7020 *
7021 * WaDisableRCCUnitClockGating:snb
7022 * WaDisableRCPBUnitClockGating:snb
7023 */
7024 I915_WRITE(GEN6_UCGCTL2,
7025 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7026 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7027
7028 /*
7029 * According to the spec the following bits should be
7030 * set in order to enable memory self-refresh and fbc:
7031 * The bit21 and bit22 of 0x42000
7032 * The bit21 and bit22 of 0x42004
7033 * The bit5 and bit7 of 0x42020
7034 * The bit14 of 0x70180
7035 * The bit14 of 0x71180
7036 *
7037 * WaFbcAsynchFlipDisableFbcQueue:snb
7038 */
7039 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7040 I915_READ(ILK_DISPLAY_CHICKEN1) |
7041 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7042 I915_WRITE(ILK_DISPLAY_CHICKEN2,
7043 I915_READ(ILK_DISPLAY_CHICKEN2) |
7044 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7045 I915_WRITE(ILK_DSPCLK_GATE_D,
7046 I915_READ(ILK_DSPCLK_GATE_D) |
7047 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7048 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7049
7050 g4x_disable_trickle_feed(dev_priv);
7051
7052 cpt_init_clock_gating(dev_priv);
7053
7054 gen6_check_mch_setup(dev_priv);
7055}
7056
7057static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7058{
7059 /*
7060 * TODO: this bit should only be enabled when really needed, then
7061 * disabled when not needed anymore in order to save power.
7062 */
7063 if (HAS_PCH_LPT_LP(dev_priv))
7064 I915_WRITE(SOUTH_DSPCLK_GATE_D,
7065 I915_READ(SOUTH_DSPCLK_GATE_D) |
7066 PCH_LP_PARTITION_LEVEL_DISABLE);
7067
7068 /* WADPOClockGatingDisable:hsw */
7069 I915_WRITE(TRANS_CHICKEN1(PIPE_A),
7070 I915_READ(TRANS_CHICKEN1(PIPE_A)) |
7071 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7072}
7073
7074static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7075{
7076 if (HAS_PCH_LPT_LP(dev_priv)) {
7077 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
7078
7079 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7080 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
7081 }
7082}
7083
7084static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7085 int general_prio_credits,
7086 int high_prio_credits)
7087{
7088 u32 misccpctl;
7089 u32 val;
7090
7091 /* WaTempDisableDOPClkGating:bdw */
7092 misccpctl = I915_READ(GEN7_MISCCPCTL);
7093 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7094
7095 val = I915_READ(GEN8_L3SQCREG1);
7096 val &= ~L3_PRIO_CREDITS_MASK;
7097 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7098 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7099 I915_WRITE(GEN8_L3SQCREG1, val);
7100
7101 /*
7102 * Wait at least 100 clocks before re-enabling clock gating.
7103 * See the definition of L3SQCREG1 in BSpec.
7104 */
7105 POSTING_READ(GEN8_L3SQCREG1);
7106 udelay(1);
7107 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
7108}
7109
7110static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7111{
7112 /* Wa_1409120013:icl,ehl */
7113 I915_WRITE(ILK_DPFC_CHICKEN,
7114 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7115
7116 /* This is not an Wa. Enable to reduce Sampler power */
7117 I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
7118 I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7119
7120 /*Wa_14010594013:icl, ehl */
7121 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7122 0, CNL_DELAY_PMRSP);
7123}
7124
7125static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
7126{
7127 u32 vd_pg_enable = 0;
7128 unsigned int i;
7129
7130 /* Wa_1409120013:tgl */
7131 I915_WRITE(ILK_DPFC_CHICKEN,
7132 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7133
7134 /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
7135 for (i = 0; i < I915_MAX_VCS; i++) {
7136 if (HAS_ENGINE(&dev_priv->gt, _VCS(i)))
7137 vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
7138 VDN_MFX_POWERGATE_ENABLE(i);
7139 }
7140
7141 I915_WRITE(POWERGATE_ENABLE,
7142 I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
7143
7144 /* Wa_1409825376:tgl (pre-prod)*/
7145 if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
7146 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
7147 TGL_VRH_GATING_DIS);
7148
7149 /* Wa_14011059788:tgl */
7150 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7151 0, DFR_DISABLE);
7152}
7153
7154static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7155{
7156 if (!HAS_PCH_CNP(dev_priv))
7157 return;
7158
7159 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7160 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
7161 CNP_PWM_CGE_GATING_DISABLE);
7162}
7163
7164static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7165{
7166 u32 val;
7167 cnp_init_clock_gating(dev_priv);
7168
7169 /* This is not an Wa. Enable for better image quality */
7170 I915_WRITE(_3D_CHICKEN3,
7171 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7172
7173 /* WaEnableChickenDCPR:cnl */
7174 I915_WRITE(GEN8_CHICKEN_DCPR_1,
7175 I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7176
7177 /*
7178 * WaFbcWakeMemOn:cnl
7179 * Display WA #0859: cnl
7180 */
7181 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7182 DISP_FBC_MEMORY_WAKE);
7183
7184 val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
7185 /* ReadHitWriteOnlyDisable:cnl */
7186 val |= RCCUNIT_CLKGATE_DIS;
7187 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
7188
7189 /* Wa_2201832410:cnl */
7190 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
7191 val |= GWUNIT_CLKGATE_DIS;
7192 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7193
7194 /* WaDisableVFclkgate:cnl */
7195 /* WaVFUnitClockGatingDisable:cnl */
7196 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
7197 val |= VFUNIT_CLKGATE_DIS;
7198 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
7199}
7200
7201static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7202{
7203 cnp_init_clock_gating(dev_priv);
7204 gen9_init_clock_gating(dev_priv);
7205
7206 /*
7207 * WaFbcTurnOffFbcWatermark:cfl
7208 * Display WA #0562: cfl
7209 */
7210 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7211 DISP_FBC_WM_DIS);
7212
7213 /*
7214 * WaFbcNukeOnHostModify:cfl
7215 * Display WA #0873: cfl
7216 */
7217 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7218 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7219}
7220
7221static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7222{
7223 gen9_init_clock_gating(dev_priv);
7224
7225 /* WaDisableSDEUnitClockGating:kbl */
7226 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7227 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7228 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7229
7230 /* WaDisableGamClockGating:kbl */
7231 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
7232 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7233 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7234
7235 /*
7236 * WaFbcTurnOffFbcWatermark:kbl
7237 * Display WA #0562: kbl
7238 */
7239 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7240 DISP_FBC_WM_DIS);
7241
7242 /*
7243 * WaFbcNukeOnHostModify:kbl
7244 * Display WA #0873: kbl
7245 */
7246 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7247 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7248}
7249
7250static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7251{
7252 gen9_init_clock_gating(dev_priv);
7253
7254 /* WAC6entrylatency:skl */
7255 I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
7256 FBC_LLC_FULLY_OPEN);
7257
7258 /*
7259 * WaFbcTurnOffFbcWatermark:skl
7260 * Display WA #0562: skl
7261 */
7262 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
7263 DISP_FBC_WM_DIS);
7264
7265 /*
7266 * WaFbcNukeOnHostModify:skl
7267 * Display WA #0873: skl
7268 */
7269 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7270 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7271
7272 /*
7273 * WaFbcHighMemBwCorruptionAvoidance:skl
7274 * Display WA #0883: skl
7275 */
7276 I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
7277 ILK_DPFC_DISABLE_DUMMY0);
7278}
7279
7280static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7281{
7282 enum pipe pipe;
7283
7284 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7285 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7286 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7287 HSW_FBCQ_DIS);
7288
7289 /* WaSwitchSolVfFArbitrationPriority:bdw */
7290 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7291
7292 /* WaPsrDPAMaskVBlankInSRD:bdw */
7293 I915_WRITE(CHICKEN_PAR1_1,
7294 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7295
7296 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7297 for_each_pipe(dev_priv, pipe) {
7298 I915_WRITE(CHICKEN_PIPESL_1(pipe),
7299 I915_READ(CHICKEN_PIPESL_1(pipe)) |
7300 BDW_DPRS_MASK_VBLANK_SRD);
7301 }
7302
7303 /* WaVSRefCountFullforceMissDisable:bdw */
7304 /* WaDSRefCountFullforceMissDisable:bdw */
7305 I915_WRITE(GEN7_FF_THREAD_MODE,
7306 I915_READ(GEN7_FF_THREAD_MODE) &
7307 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7308
7309 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7310 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7311
7312 /* WaDisableSDEUnitClockGating:bdw */
7313 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7314 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7315
7316 /* WaProgramL3SqcReg1Default:bdw */
7317 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7318
7319 /* WaKVMNotificationOnConfigChange:bdw */
7320 I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
7321 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7322
7323 lpt_init_clock_gating(dev_priv);
7324
7325 /* WaDisableDopClockGating:bdw
7326 *
7327 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7328 * clock gating.
7329 */
7330 I915_WRITE(GEN6_UCGCTL1,
7331 I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7332}
7333
7334static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7335{
7336 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7337 I915_WRITE(CHICKEN_PIPESL_1(PIPE_A),
7338 I915_READ(CHICKEN_PIPESL_1(PIPE_A)) |
7339 HSW_FBCQ_DIS);
7340
7341 /* This is required by WaCatErrorRejectionIssue:hsw */
7342 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7343 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7344 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7345
7346 /* WaSwitchSolVfFArbitrationPriority:hsw */
7347 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7348
7349 lpt_init_clock_gating(dev_priv);
7350}
7351
7352static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7353{
7354 u32 snpcr;
7355
7356 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7357
7358 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7359 I915_WRITE(ILK_DISPLAY_CHICKEN1,
7360 I915_READ(ILK_DISPLAY_CHICKEN1) |
7361 ILK_FBCQ_DIS);
7362
7363 /* WaDisableBackToBackFlipFix:ivb */
7364 I915_WRITE(IVB_CHICKEN3,
7365 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7366 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7367
7368 if (IS_IVB_GT1(dev_priv))
7369 I915_WRITE(GEN7_ROW_CHICKEN2,
7370 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7371 else {
7372 /* must write both registers */
7373 I915_WRITE(GEN7_ROW_CHICKEN2,
7374 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7375 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7376 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7377 }
7378
7379 /*
7380 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7381 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7382 */
7383 I915_WRITE(GEN6_UCGCTL2,
7384 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7385
7386 /* This is required by WaCatErrorRejectionIssue:ivb */
7387 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7388 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7389 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7390
7391 g4x_disable_trickle_feed(dev_priv);
7392
7393 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7394 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7395 snpcr |= GEN6_MBC_SNPCR_MED;
7396 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7397
7398 if (!HAS_PCH_NOP(dev_priv))
7399 cpt_init_clock_gating(dev_priv);
7400
7401 gen6_check_mch_setup(dev_priv);
7402}
7403
7404static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7405{
7406 /* WaDisableBackToBackFlipFix:vlv */
7407 I915_WRITE(IVB_CHICKEN3,
7408 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7409 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7410
7411 /* WaDisableDopClockGating:vlv */
7412 I915_WRITE(GEN7_ROW_CHICKEN2,
7413 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7414
7415 /* This is required by WaCatErrorRejectionIssue:vlv */
7416 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7417 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7418 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7419
7420 /*
7421 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7422 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7423 */
7424 I915_WRITE(GEN6_UCGCTL2,
7425 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7426
7427 /* WaDisableL3Bank2xClockGate:vlv
7428 * Disabling L3 clock gating- MMIO 940c[25] = 1
7429 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7430 I915_WRITE(GEN7_UCGCTL4,
7431 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7432
7433 /*
7434 * WaDisableVLVClockGating_VBIIssue:vlv
7435 * Disable clock gating on th GCFG unit to prevent a delay
7436 * in the reporting of vblank events.
7437 */
7438 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7439}
7440
7441static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7442{
7443 /* WaVSRefCountFullforceMissDisable:chv */
7444 /* WaDSRefCountFullforceMissDisable:chv */
7445 I915_WRITE(GEN7_FF_THREAD_MODE,
7446 I915_READ(GEN7_FF_THREAD_MODE) &
7447 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7448
7449 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7450 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7451 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7452
7453 /* WaDisableCSUnitClockGating:chv */
7454 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7455 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7456
7457 /* WaDisableSDEUnitClockGating:chv */
7458 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7459 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7460
7461 /*
7462 * WaProgramL3SqcReg1Default:chv
7463 * See gfxspecs/Related Documents/Performance Guide/
7464 * LSQC Setting Recommendations.
7465 */
7466 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7467}
7468
7469static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7470{
7471 u32 dspclk_gate;
7472
7473 I915_WRITE(RENCLK_GATE_D1, 0);
7474 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7475 GS_UNIT_CLOCK_GATE_DISABLE |
7476 CL_UNIT_CLOCK_GATE_DISABLE);
7477 I915_WRITE(RAMCLK_GATE_D, 0);
7478 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7479 OVRUNIT_CLOCK_GATE_DISABLE |
7480 OVCUNIT_CLOCK_GATE_DISABLE;
7481 if (IS_GM45(dev_priv))
7482 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7483 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7484
7485 g4x_disable_trickle_feed(dev_priv);
7486}
7487
7488static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7489{
7490 struct intel_uncore *uncore = &dev_priv->uncore;
7491
7492 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7493 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7494 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7495 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7496 intel_uncore_write16(uncore, DEUC, 0);
7497 intel_uncore_write(uncore,
7498 MI_ARB_STATE,
7499 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7500}
7501
7502static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7503{
7504 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7505 I965_RCC_CLOCK_GATE_DISABLE |
7506 I965_RCPB_CLOCK_GATE_DISABLE |
7507 I965_ISC_CLOCK_GATE_DISABLE |
7508 I965_FBC_CLOCK_GATE_DISABLE);
7509 I915_WRITE(RENCLK_GATE_D2, 0);
7510 I915_WRITE(MI_ARB_STATE,
7511 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7512}
7513
7514static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7515{
7516 u32 dstate = I915_READ(D_STATE);
7517
7518 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7519 DSTATE_DOT_CLOCK_GATING;
7520 I915_WRITE(D_STATE, dstate);
7521
7522 if (IS_PINEVIEW(dev_priv))
7523 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7524
7525 /* IIR "flip pending" means done if this bit is set */
7526 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7527
7528 /* interrupts should cause a wake up from C3 */
7529 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7530
7531 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7532 I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7533
7534 I915_WRITE(MI_ARB_STATE,
7535 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7536}
7537
7538static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7539{
7540 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7541
7542 /* interrupts should cause a wake up from C3 */
7543 I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7544 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7545
7546 I915_WRITE(MEM_MODE,
7547 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7548
7549 /*
7550 * Have FBC ignore 3D activity since we use software
7551 * render tracking, and otherwise a pure 3D workload
7552 * (even if it just renders a single frame and then does
7553 * abosultely nothing) would not allow FBC to recompress
7554 * until a 2D blit occurs.
7555 */
7556 I915_WRITE(SCPD0,
7557 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7558}
7559
7560static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7561{
7562 I915_WRITE(MEM_MODE,
7563 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7564 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7565}
7566
7567void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7568{
7569 dev_priv->display.init_clock_gating(dev_priv);
7570}
7571
7572void intel_suspend_hw(struct drm_i915_private *dev_priv)
7573{
7574 if (HAS_PCH_LPT(dev_priv))
7575 lpt_suspend_hw(dev_priv);
7576}
7577
7578static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7579{
7580 drm_dbg_kms(&dev_priv->drm,
7581 "No clock gating settings or workarounds applied.\n");
7582}
7583
7584/**
7585 * intel_init_clock_gating_hooks - setup the clock gating hooks
7586 * @dev_priv: device private
7587 *
7588 * Setup the hooks that configure which clocks of a given platform can be
7589 * gated and also apply various GT and display specific workarounds for these
7590 * platforms. Note that some GT specific workarounds are applied separately
7591 * when GPU contexts or batchbuffers start their execution.
7592 */
7593void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7594{
7595 if (IS_GEN(dev_priv, 12))
7596 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7597 else if (IS_GEN(dev_priv, 11))
7598 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7599 else if (IS_CANNONLAKE(dev_priv))
7600 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7601 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7602 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7603 else if (IS_SKYLAKE(dev_priv))
7604 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7605 else if (IS_KABYLAKE(dev_priv))
7606 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7607 else if (IS_BROXTON(dev_priv))
7608 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7609 else if (IS_GEMINILAKE(dev_priv))
7610 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7611 else if (IS_BROADWELL(dev_priv))
7612 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7613 else if (IS_CHERRYVIEW(dev_priv))
7614 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7615 else if (IS_HASWELL(dev_priv))
7616 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7617 else if (IS_IVYBRIDGE(dev_priv))
7618 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7619 else if (IS_VALLEYVIEW(dev_priv))
7620 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7621 else if (IS_GEN(dev_priv, 6))
7622 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7623 else if (IS_GEN(dev_priv, 5))
7624 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7625 else if (IS_G4X(dev_priv))
7626 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7627 else if (IS_I965GM(dev_priv))
7628 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7629 else if (IS_I965G(dev_priv))
7630 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7631 else if (IS_GEN(dev_priv, 3))
7632 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7633 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7634 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7635 else if (IS_GEN(dev_priv, 2))
7636 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7637 else {
7638 MISSING_CASE(INTEL_DEVID(dev_priv));
7639 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7640 }
7641}
7642
7643/* Set up chip specific power management-related functions */
7644void intel_init_pm(struct drm_i915_private *dev_priv)
7645{
7646 /* For cxsr */
7647 if (IS_PINEVIEW(dev_priv))
7648 pnv_get_mem_freq(dev_priv);
7649 else if (IS_GEN(dev_priv, 5))
7650 ilk_get_mem_freq(dev_priv);
7651
7652 if (intel_has_sagv(dev_priv))
7653 skl_setup_sagv_block_time(dev_priv);
7654
7655 /* For FIFO watermark updates */
7656 if (INTEL_GEN(dev_priv) >= 9) {
7657 skl_setup_wm_latency(dev_priv);
7658 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7659 } else if (HAS_PCH_SPLIT(dev_priv)) {
7660 ilk_setup_wm_latency(dev_priv);
7661
7662 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7663 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7664 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7665 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7666 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7667 dev_priv->display.compute_intermediate_wm =
7668 ilk_compute_intermediate_wm;
7669 dev_priv->display.initial_watermarks =
7670 ilk_initial_watermarks;
7671 dev_priv->display.optimize_watermarks =
7672 ilk_optimize_watermarks;
7673 } else {
7674 drm_dbg_kms(&dev_priv->drm,
7675 "Failed to read display plane latency. "
7676 "Disable CxSR\n");
7677 }
7678 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7679 vlv_setup_wm_latency(dev_priv);
7680 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7681 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7682 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7683 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7684 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7685 } else if (IS_G4X(dev_priv)) {
7686 g4x_setup_wm_latency(dev_priv);
7687 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7688 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7689 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7690 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7691 } else if (IS_PINEVIEW(dev_priv)) {
7692 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7693 dev_priv->is_ddr3,
7694 dev_priv->fsb_freq,
7695 dev_priv->mem_freq)) {
7696 drm_info(&dev_priv->drm,
7697 "failed to find known CxSR latency "
7698 "(found ddr%s fsb freq %d, mem freq %d), "
7699 "disabling CxSR\n",
7700 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7701 dev_priv->fsb_freq, dev_priv->mem_freq);
7702 /* Disable CxSR and never update its watermark again */
7703 intel_set_memory_cxsr(dev_priv, false);
7704 dev_priv->display.update_wm = NULL;
7705 } else
7706 dev_priv->display.update_wm = pnv_update_wm;
7707 } else if (IS_GEN(dev_priv, 4)) {
7708 dev_priv->display.update_wm = i965_update_wm;
7709 } else if (IS_GEN(dev_priv, 3)) {
7710 dev_priv->display.update_wm = i9xx_update_wm;
7711 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7712 } else if (IS_GEN(dev_priv, 2)) {
7713 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7714 dev_priv->display.update_wm = i845_update_wm;
7715 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7716 } else {
7717 dev_priv->display.update_wm = i9xx_update_wm;
7718 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7719 }
7720 } else {
7721 drm_err(&dev_priv->drm,
7722 "unexpected fall-through in %s\n", __func__);
7723 }
7724}
7725
7726void intel_pm_setup(struct drm_i915_private *dev_priv)
7727{
7728 dev_priv->runtime_pm.suspended = false;
7729 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7730}
7731
7732static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7733{
7734 struct intel_dbuf_state *dbuf_state;
7735
7736 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7737 if (!dbuf_state)
7738 return NULL;
7739
7740 return &dbuf_state->base;
7741}
7742
7743static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7744 struct intel_global_state *state)
7745{
7746 kfree(state);
7747}
7748
7749static const struct intel_global_state_funcs intel_dbuf_funcs = {
7750 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7751 .atomic_destroy_state = intel_dbuf_destroy_state,
7752};
7753
7754struct intel_dbuf_state *
7755intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7756{
7757 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7758 struct intel_global_state *dbuf_state;
7759
7760 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7761 if (IS_ERR(dbuf_state))
7762 return ERR_CAST(dbuf_state);
7763
7764 return to_intel_dbuf_state(dbuf_state);
7765}
7766
7767int intel_dbuf_init(struct drm_i915_private *dev_priv)
7768{
7769 struct intel_dbuf_state *dbuf_state;
7770
7771 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7772 if (!dbuf_state)
7773 return -ENOMEM;
7774
7775 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7776 &dbuf_state->base, &intel_dbuf_funcs);
7777
7778 return 0;
7779}
7780
7781void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7782{
7783 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7784 const struct intel_dbuf_state *new_dbuf_state =
7785 intel_atomic_get_new_dbuf_state(state);
7786 const struct intel_dbuf_state *old_dbuf_state =
7787 intel_atomic_get_old_dbuf_state(state);
7788
7789 if (!new_dbuf_state ||
7790 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7791 return;
7792
7793 WARN_ON(!new_dbuf_state->base.changed);
7794
7795 gen9_dbuf_slices_update(dev_priv,
7796 old_dbuf_state->enabled_slices |
7797 new_dbuf_state->enabled_slices);
7798}
7799
7800void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7801{
7802 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7803 const struct intel_dbuf_state *new_dbuf_state =
7804 intel_atomic_get_new_dbuf_state(state);
7805 const struct intel_dbuf_state *old_dbuf_state =
7806 intel_atomic_get_old_dbuf_state(state);
7807
7808 if (!new_dbuf_state ||
7809 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7810 return;
7811
7812 WARN_ON(!new_dbuf_state->base.changed);
7813
7814 gen9_dbuf_slices_update(dev_priv,
7815 new_dbuf_state->enabled_slices);
7816}