Loading...
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include "display/intel_de.h"
29#include "display/intel_display_trace.h"
30#include "display/skl_watermark.h"
31
32#include "gt/intel_engine_regs.h"
33#include "gt/intel_gt.h"
34#include "gt/intel_gt_mcr.h"
35#include "gt/intel_gt_regs.h"
36
37#include "i915_drv.h"
38#include "intel_mchbar_regs.h"
39#include "intel_pm.h"
40#include "vlv_sideband.h"
41
42struct drm_i915_clock_gating_funcs {
43 void (*init_clock_gating)(struct drm_i915_private *i915);
44};
45
46/* used in computing the new watermarks state */
47struct intel_wm_config {
48 unsigned int num_pipes_active;
49 bool sprites_enabled;
50 bool sprites_scaled;
51};
52
53static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
54{
55 if (HAS_LLC(dev_priv)) {
56 /*
57 * WaCompressedResourceDisplayNewHashMode:skl,kbl
58 * Display WA #0390: skl,kbl
59 *
60 * Must match Sampler, Pixel Back End, and Media. See
61 * WaCompressedResourceSamplerPbeMediaNewHashMode.
62 */
63 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
64 }
65
66 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
67 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
68
69 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
70 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
71
72 /*
73 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
74 * Display WA #0859: skl,bxt,kbl,glk,cfl
75 */
76 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
77}
78
79static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
80{
81 gen9_init_clock_gating(dev_priv);
82
83 /* WaDisableSDEUnitClockGating:bxt */
84 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
85
86 /*
87 * FIXME:
88 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
89 */
90 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
91
92 /*
93 * Wa: Backlight PWM may stop in the asserted state, causing backlight
94 * to stay fully on.
95 */
96 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
97 PWM1_GATING_DIS | PWM2_GATING_DIS);
98
99 /*
100 * Lower the display internal timeout.
101 * This is needed to avoid any hard hangs when DSI port PLL
102 * is off and a MMIO access is attempted by any privilege
103 * application, using batch buffers or any other means.
104 */
105 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
106
107 /*
108 * WaFbcTurnOffFbcWatermark:bxt
109 * Display WA #0562: bxt
110 */
111 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
112
113 /*
114 * WaFbcHighMemBwCorruptionAvoidance:bxt
115 * Display WA #0883: bxt
116 */
117 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
118}
119
120static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124 /*
125 * WaDisablePWMClockGating:glk
126 * Backlight PWM may stop in the asserted state, causing backlight
127 * to stay fully on.
128 */
129 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
130 PWM1_GATING_DIS | PWM2_GATING_DIS);
131}
132
133static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
134{
135 u32 tmp;
136
137 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
138
139 switch (tmp & CLKCFG_FSB_MASK) {
140 case CLKCFG_FSB_533:
141 dev_priv->fsb_freq = 533; /* 133*4 */
142 break;
143 case CLKCFG_FSB_800:
144 dev_priv->fsb_freq = 800; /* 200*4 */
145 break;
146 case CLKCFG_FSB_667:
147 dev_priv->fsb_freq = 667; /* 167*4 */
148 break;
149 case CLKCFG_FSB_400:
150 dev_priv->fsb_freq = 400; /* 100*4 */
151 break;
152 }
153
154 switch (tmp & CLKCFG_MEM_MASK) {
155 case CLKCFG_MEM_533:
156 dev_priv->mem_freq = 533;
157 break;
158 case CLKCFG_MEM_667:
159 dev_priv->mem_freq = 667;
160 break;
161 case CLKCFG_MEM_800:
162 dev_priv->mem_freq = 800;
163 break;
164 }
165
166 /* detect pineview DDR3 setting */
167 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
168 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
169}
170
171static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
172{
173 u16 ddrpll, csipll;
174
175 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
176 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
177
178 switch (ddrpll & 0xff) {
179 case 0xc:
180 dev_priv->mem_freq = 800;
181 break;
182 case 0x10:
183 dev_priv->mem_freq = 1066;
184 break;
185 case 0x14:
186 dev_priv->mem_freq = 1333;
187 break;
188 case 0x18:
189 dev_priv->mem_freq = 1600;
190 break;
191 default:
192 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
193 ddrpll & 0xff);
194 dev_priv->mem_freq = 0;
195 break;
196 }
197
198 switch (csipll & 0x3ff) {
199 case 0x00c:
200 dev_priv->fsb_freq = 3200;
201 break;
202 case 0x00e:
203 dev_priv->fsb_freq = 3733;
204 break;
205 case 0x010:
206 dev_priv->fsb_freq = 4266;
207 break;
208 case 0x012:
209 dev_priv->fsb_freq = 4800;
210 break;
211 case 0x014:
212 dev_priv->fsb_freq = 5333;
213 break;
214 case 0x016:
215 dev_priv->fsb_freq = 5866;
216 break;
217 case 0x018:
218 dev_priv->fsb_freq = 6400;
219 break;
220 default:
221 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
222 csipll & 0x3ff);
223 dev_priv->fsb_freq = 0;
224 break;
225 }
226}
227
228static const struct cxsr_latency cxsr_latency_table[] = {
229 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
230 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
231 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
232 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
233 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
234
235 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
236 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
237 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
238 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
239 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
240
241 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
242 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
243 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
244 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
245 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
246
247 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
248 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
249 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
250 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
251 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
252
253 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
254 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
255 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
256 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
257 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
258
259 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
260 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
261 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
262 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
263 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
264};
265
266static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
267 bool is_ddr3,
268 int fsb,
269 int mem)
270{
271 const struct cxsr_latency *latency;
272 int i;
273
274 if (fsb == 0 || mem == 0)
275 return NULL;
276
277 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
278 latency = &cxsr_latency_table[i];
279 if (is_desktop == latency->is_desktop &&
280 is_ddr3 == latency->is_ddr3 &&
281 fsb == latency->fsb_freq && mem == latency->mem_freq)
282 return latency;
283 }
284
285 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
286
287 return NULL;
288}
289
290static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
291{
292 u32 val;
293
294 vlv_punit_get(dev_priv);
295
296 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
297 if (enable)
298 val &= ~FORCE_DDR_HIGH_FREQ;
299 else
300 val |= FORCE_DDR_HIGH_FREQ;
301 val &= ~FORCE_DDR_LOW_FREQ;
302 val |= FORCE_DDR_FREQ_REQ_ACK;
303 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
304
305 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
306 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
307 drm_err(&dev_priv->drm,
308 "timed out waiting for Punit DDR DVFS request\n");
309
310 vlv_punit_put(dev_priv);
311}
312
313static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
314{
315 u32 val;
316
317 vlv_punit_get(dev_priv);
318
319 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
320 if (enable)
321 val |= DSP_MAXFIFO_PM5_ENABLE;
322 else
323 val &= ~DSP_MAXFIFO_PM5_ENABLE;
324 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
325
326 vlv_punit_put(dev_priv);
327}
328
329#define FW_WM(value, plane) \
330 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
331
332static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
333{
334 bool was_enabled;
335 u32 val;
336
337 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
338 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
339 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
340 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
341 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
342 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
343 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
344 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
345 } else if (IS_PINEVIEW(dev_priv)) {
346 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
347 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
348 if (enable)
349 val |= PINEVIEW_SELF_REFRESH_EN;
350 else
351 val &= ~PINEVIEW_SELF_REFRESH_EN;
352 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
353 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
354 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
355 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
356 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
357 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
358 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
359 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
360 } else if (IS_I915GM(dev_priv)) {
361 /*
362 * FIXME can't find a bit like this for 915G, and
363 * and yet it does have the related watermark in
364 * FW_BLC_SELF. What's going on?
365 */
366 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
367 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
368 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
369 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
370 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
371 } else {
372 return false;
373 }
374
375 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
376
377 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
378 str_enabled_disabled(enable),
379 str_enabled_disabled(was_enabled));
380
381 return was_enabled;
382}
383
384/**
385 * intel_set_memory_cxsr - Configure CxSR state
386 * @dev_priv: i915 device
387 * @enable: Allow vs. disallow CxSR
388 *
389 * Allow or disallow the system to enter a special CxSR
390 * (C-state self refresh) state. What typically happens in CxSR mode
391 * is that several display FIFOs may get combined into a single larger
392 * FIFO for a particular plane (so called max FIFO mode) to allow the
393 * system to defer memory fetches longer, and the memory will enter
394 * self refresh.
395 *
396 * Note that enabling CxSR does not guarantee that the system enter
397 * this special mode, nor does it guarantee that the system stays
398 * in that mode once entered. So this just allows/disallows the system
399 * to autonomously utilize the CxSR mode. Other factors such as core
400 * C-states will affect when/if the system actually enters/exits the
401 * CxSR mode.
402 *
403 * Note that on VLV/CHV this actually only controls the max FIFO mode,
404 * and the system is free to enter/exit memory self refresh at any time
405 * even when the use of CxSR has been disallowed.
406 *
407 * While the system is actually in the CxSR/max FIFO mode, some plane
408 * control registers will not get latched on vblank. Thus in order to
409 * guarantee the system will respond to changes in the plane registers
410 * we must always disallow CxSR prior to making changes to those registers.
411 * Unfortunately the system will re-evaluate the CxSR conditions at
412 * frame start which happens after vblank start (which is when the plane
413 * registers would get latched), so we can't proceed with the plane update
414 * during the same frame where we disallowed CxSR.
415 *
416 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
417 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
418 * the hardware w.r.t. HPLL SR when writing to plane registers.
419 * Disallowing just CxSR is sufficient.
420 */
421bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
422{
423 bool ret;
424
425 mutex_lock(&dev_priv->display.wm.wm_mutex);
426 ret = _intel_set_memory_cxsr(dev_priv, enable);
427 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
428 dev_priv->display.wm.vlv.cxsr = enable;
429 else if (IS_G4X(dev_priv))
430 dev_priv->display.wm.g4x.cxsr = enable;
431 mutex_unlock(&dev_priv->display.wm.wm_mutex);
432
433 return ret;
434}
435
436/*
437 * Latency for FIFO fetches is dependent on several factors:
438 * - memory configuration (speed, channels)
439 * - chipset
440 * - current MCH state
441 * It can be fairly high in some situations, so here we assume a fairly
442 * pessimal value. It's a tradeoff between extra memory fetches (if we
443 * set this value too high, the FIFO will fetch frequently to stay full)
444 * and power consumption (set it too low to save power and we might see
445 * FIFO underruns and display "flicker").
446 *
447 * A value of 5us seems to be a good balance; safe for very low end
448 * platforms but not overly aggressive on lower latency configs.
449 */
450static const int pessimal_latency_ns = 5000;
451
452#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
453 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
454
455static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
456{
457 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
459 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
460 enum pipe pipe = crtc->pipe;
461 int sprite0_start, sprite1_start;
462 u32 dsparb, dsparb2, dsparb3;
463
464 switch (pipe) {
465 case PIPE_A:
466 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
467 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
468 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
469 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
470 break;
471 case PIPE_B:
472 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
473 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
474 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
475 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
476 break;
477 case PIPE_C:
478 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
479 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
480 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
481 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
482 break;
483 default:
484 MISSING_CASE(pipe);
485 return;
486 }
487
488 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
489 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
490 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
491 fifo_state->plane[PLANE_CURSOR] = 63;
492}
493
494static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
495 enum i9xx_plane_id i9xx_plane)
496{
497 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
498 int size;
499
500 size = dsparb & 0x7f;
501 if (i9xx_plane == PLANE_B)
502 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
503
504 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
505 dsparb, plane_name(i9xx_plane), size);
506
507 return size;
508}
509
510static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
511 enum i9xx_plane_id i9xx_plane)
512{
513 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
514 int size;
515
516 size = dsparb & 0x1ff;
517 if (i9xx_plane == PLANE_B)
518 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
519 size >>= 1; /* Convert to cachelines */
520
521 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
522 dsparb, plane_name(i9xx_plane), size);
523
524 return size;
525}
526
527static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
528 enum i9xx_plane_id i9xx_plane)
529{
530 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
531 int size;
532
533 size = dsparb & 0x7f;
534 size >>= 2; /* Convert to cachelines */
535
536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
538
539 return size;
540}
541
542/* Pineview has different values for various configs */
543static const struct intel_watermark_params pnv_display_wm = {
544 .fifo_size = PINEVIEW_DISPLAY_FIFO,
545 .max_wm = PINEVIEW_MAX_WM,
546 .default_wm = PINEVIEW_DFT_WM,
547 .guard_size = PINEVIEW_GUARD_WM,
548 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
549};
550
551static const struct intel_watermark_params pnv_display_hplloff_wm = {
552 .fifo_size = PINEVIEW_DISPLAY_FIFO,
553 .max_wm = PINEVIEW_MAX_WM,
554 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
555 .guard_size = PINEVIEW_GUARD_WM,
556 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
557};
558
559static const struct intel_watermark_params pnv_cursor_wm = {
560 .fifo_size = PINEVIEW_CURSOR_FIFO,
561 .max_wm = PINEVIEW_CURSOR_MAX_WM,
562 .default_wm = PINEVIEW_CURSOR_DFT_WM,
563 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
564 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
565};
566
567static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
568 .fifo_size = PINEVIEW_CURSOR_FIFO,
569 .max_wm = PINEVIEW_CURSOR_MAX_WM,
570 .default_wm = PINEVIEW_CURSOR_DFT_WM,
571 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
572 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
573};
574
575static const struct intel_watermark_params i965_cursor_wm_info = {
576 .fifo_size = I965_CURSOR_FIFO,
577 .max_wm = I965_CURSOR_MAX_WM,
578 .default_wm = I965_CURSOR_DFT_WM,
579 .guard_size = 2,
580 .cacheline_size = I915_FIFO_LINE_SIZE,
581};
582
583static const struct intel_watermark_params i945_wm_info = {
584 .fifo_size = I945_FIFO_SIZE,
585 .max_wm = I915_MAX_WM,
586 .default_wm = 1,
587 .guard_size = 2,
588 .cacheline_size = I915_FIFO_LINE_SIZE,
589};
590
591static const struct intel_watermark_params i915_wm_info = {
592 .fifo_size = I915_FIFO_SIZE,
593 .max_wm = I915_MAX_WM,
594 .default_wm = 1,
595 .guard_size = 2,
596 .cacheline_size = I915_FIFO_LINE_SIZE,
597};
598
599static const struct intel_watermark_params i830_a_wm_info = {
600 .fifo_size = I855GM_FIFO_SIZE,
601 .max_wm = I915_MAX_WM,
602 .default_wm = 1,
603 .guard_size = 2,
604 .cacheline_size = I830_FIFO_LINE_SIZE,
605};
606
607static const struct intel_watermark_params i830_bc_wm_info = {
608 .fifo_size = I855GM_FIFO_SIZE,
609 .max_wm = I915_MAX_WM/2,
610 .default_wm = 1,
611 .guard_size = 2,
612 .cacheline_size = I830_FIFO_LINE_SIZE,
613};
614
615static const struct intel_watermark_params i845_wm_info = {
616 .fifo_size = I830_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
618 .default_wm = 1,
619 .guard_size = 2,
620 .cacheline_size = I830_FIFO_LINE_SIZE,
621};
622
623/**
624 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
625 * @pixel_rate: Pipe pixel rate in kHz
626 * @cpp: Plane bytes per pixel
627 * @latency: Memory wakeup latency in 0.1us units
628 *
629 * Compute the watermark using the method 1 or "small buffer"
630 * formula. The caller may additonally add extra cachelines
631 * to account for TLB misses and clock crossings.
632 *
633 * This method is concerned with the short term drain rate
634 * of the FIFO, ie. it does not account for blanking periods
635 * which would effectively reduce the average drain rate across
636 * a longer period. The name "small" refers to the fact the
637 * FIFO is relatively small compared to the amount of data
638 * fetched.
639 *
640 * The FIFO level vs. time graph might look something like:
641 *
642 * |\ |\
643 * | \ | \
644 * __---__---__ (- plane active, _ blanking)
645 * -> time
646 *
647 * or perhaps like this:
648 *
649 * |\|\ |\|\
650 * __----__----__ (- plane active, _ blanking)
651 * -> time
652 *
653 * Returns:
654 * The watermark in bytes
655 */
656static unsigned int intel_wm_method1(unsigned int pixel_rate,
657 unsigned int cpp,
658 unsigned int latency)
659{
660 u64 ret;
661
662 ret = mul_u32_u32(pixel_rate, cpp * latency);
663 ret = DIV_ROUND_UP_ULL(ret, 10000);
664
665 return ret;
666}
667
668/**
669 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @htotal: Pipe horizontal total
672 * @width: Plane width in pixels
673 * @cpp: Plane bytes per pixel
674 * @latency: Memory wakeup latency in 0.1us units
675 *
676 * Compute the watermark using the method 2 or "large buffer"
677 * formula. The caller may additonally add extra cachelines
678 * to account for TLB misses and clock crossings.
679 *
680 * This method is concerned with the long term drain rate
681 * of the FIFO, ie. it does account for blanking periods
682 * which effectively reduce the average drain rate across
683 * a longer period. The name "large" refers to the fact the
684 * FIFO is relatively large compared to the amount of data
685 * fetched.
686 *
687 * The FIFO level vs. time graph might look something like:
688 *
689 * |\___ |\___
690 * | \___ | \___
691 * | \ | \
692 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
693 * -> time
694 *
695 * Returns:
696 * The watermark in bytes
697 */
698static unsigned int intel_wm_method2(unsigned int pixel_rate,
699 unsigned int htotal,
700 unsigned int width,
701 unsigned int cpp,
702 unsigned int latency)
703{
704 unsigned int ret;
705
706 /*
707 * FIXME remove once all users are computing
708 * watermarks in the correct place.
709 */
710 if (WARN_ON_ONCE(htotal == 0))
711 htotal = 1;
712
713 ret = (latency * pixel_rate) / (htotal * 10000);
714 ret = (ret + 1) * width * cpp;
715
716 return ret;
717}
718
719/**
720 * intel_calculate_wm - calculate watermark level
721 * @pixel_rate: pixel clock
722 * @wm: chip FIFO params
723 * @fifo_size: size of the FIFO buffer
724 * @cpp: bytes per pixel
725 * @latency_ns: memory latency for the platform
726 *
727 * Calculate the watermark level (the level at which the display plane will
728 * start fetching from memory again). Each chip has a different display
729 * FIFO size and allocation, so the caller needs to figure that out and pass
730 * in the correct intel_watermark_params structure.
731 *
732 * As the pixel clock runs, the FIFO will be drained at a rate that depends
733 * on the pixel size. When it reaches the watermark level, it'll start
734 * fetching FIFO line sized based chunks from memory until the FIFO fills
735 * past the watermark point. If the FIFO drains completely, a FIFO underrun
736 * will occur, and a display engine hang could result.
737 */
738static unsigned int intel_calculate_wm(int pixel_rate,
739 const struct intel_watermark_params *wm,
740 int fifo_size, int cpp,
741 unsigned int latency_ns)
742{
743 int entries, wm_size;
744
745 /*
746 * Note: we need to make sure we don't overflow for various clock &
747 * latency values.
748 * clocks go from a few thousand to several hundred thousand.
749 * latency is usually a few thousand
750 */
751 entries = intel_wm_method1(pixel_rate, cpp,
752 latency_ns / 100);
753 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
754 wm->guard_size;
755 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
756
757 wm_size = fifo_size - entries;
758 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
759
760 /* Don't promote wm_size to unsigned... */
761 if (wm_size > wm->max_wm)
762 wm_size = wm->max_wm;
763 if (wm_size <= 0)
764 wm_size = wm->default_wm;
765
766 /*
767 * Bspec seems to indicate that the value shouldn't be lower than
768 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
769 * Lets go for 8 which is the burst size since certain platforms
770 * already use a hardcoded 8 (which is what the spec says should be
771 * done).
772 */
773 if (wm_size <= 8)
774 wm_size = 8;
775
776 return wm_size;
777}
778
779static bool is_disabling(int old, int new, int threshold)
780{
781 return old >= threshold && new < threshold;
782}
783
784static bool is_enabling(int old, int new, int threshold)
785{
786 return old < threshold && new >= threshold;
787}
788
789static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
790{
791 return dev_priv->display.wm.max_level + 1;
792}
793
794bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
795 const struct intel_plane_state *plane_state)
796{
797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
798
799 /* FIXME check the 'enable' instead */
800 if (!crtc_state->hw.active)
801 return false;
802
803 /*
804 * Treat cursor with fb as always visible since cursor updates
805 * can happen faster than the vrefresh rate, and the current
806 * watermark code doesn't handle that correctly. Cursor updates
807 * which set/clear the fb or change the cursor size are going
808 * to get throttled by intel_legacy_cursor_update() to work
809 * around this problem with the watermark code.
810 */
811 if (plane->id == PLANE_CURSOR)
812 return plane_state->hw.fb != NULL;
813 else
814 return plane_state->uapi.visible;
815}
816
817static bool intel_crtc_active(struct intel_crtc *crtc)
818{
819 /* Be paranoid as we can arrive here with only partial
820 * state retrieved from the hardware during setup.
821 *
822 * We can ditch the adjusted_mode.crtc_clock check as soon
823 * as Haswell has gained clock readout/fastboot support.
824 *
825 * We can ditch the crtc->primary->state->fb check as soon as we can
826 * properly reconstruct framebuffers.
827 *
828 * FIXME: The intel_crtc->active here should be switched to
829 * crtc->state->active once we have proper CRTC states wired up
830 * for atomic.
831 */
832 return crtc && crtc->active && crtc->base.primary->state->fb &&
833 crtc->config->hw.adjusted_mode.crtc_clock;
834}
835
836static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
837{
838 struct intel_crtc *crtc, *enabled = NULL;
839
840 for_each_intel_crtc(&dev_priv->drm, crtc) {
841 if (intel_crtc_active(crtc)) {
842 if (enabled)
843 return NULL;
844 enabled = crtc;
845 }
846 }
847
848 return enabled;
849}
850
851static void pnv_update_wm(struct drm_i915_private *dev_priv)
852{
853 struct intel_crtc *crtc;
854 const struct cxsr_latency *latency;
855 u32 reg;
856 unsigned int wm;
857
858 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
859 dev_priv->is_ddr3,
860 dev_priv->fsb_freq,
861 dev_priv->mem_freq);
862 if (!latency) {
863 drm_dbg_kms(&dev_priv->drm,
864 "Unknown FSB/MEM found, disable CxSR\n");
865 intel_set_memory_cxsr(dev_priv, false);
866 return;
867 }
868
869 crtc = single_enabled_crtc(dev_priv);
870 if (crtc) {
871 const struct drm_framebuffer *fb =
872 crtc->base.primary->state->fb;
873 int pixel_rate = crtc->config->pixel_rate;
874 int cpp = fb->format->cpp[0];
875
876 /* Display SR */
877 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
878 pnv_display_wm.fifo_size,
879 cpp, latency->display_sr);
880 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
881 reg &= ~DSPFW_SR_MASK;
882 reg |= FW_WM(wm, SR);
883 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
884 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
885
886 /* cursor SR */
887 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
888 pnv_display_wm.fifo_size,
889 4, latency->cursor_sr);
890 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
891 FW_WM(wm, CURSOR_SR));
892
893 /* Display HPLL off SR */
894 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
895 pnv_display_hplloff_wm.fifo_size,
896 cpp, latency->display_hpll_disable);
897 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
898
899 /* cursor HPLL off SR */
900 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
901 pnv_display_hplloff_wm.fifo_size,
902 4, latency->cursor_hpll_disable);
903 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
904 reg &= ~DSPFW_HPLL_CURSOR_MASK;
905 reg |= FW_WM(wm, HPLL_CURSOR);
906 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
907 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
908
909 intel_set_memory_cxsr(dev_priv, true);
910 } else {
911 intel_set_memory_cxsr(dev_priv, false);
912 }
913}
914
915/*
916 * Documentation says:
917 * "If the line size is small, the TLB fetches can get in the way of the
918 * data fetches, causing some lag in the pixel data return which is not
919 * accounted for in the above formulas. The following adjustment only
920 * needs to be applied if eight whole lines fit in the buffer at once.
921 * The WM is adjusted upwards by the difference between the FIFO size
922 * and the size of 8 whole lines. This adjustment is always performed
923 * in the actual pixel depth regardless of whether FBC is enabled or not."
924 */
925static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
926{
927 int tlb_miss = fifo_size * 64 - width * cpp * 8;
928
929 return max(0, tlb_miss);
930}
931
932static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
933 const struct g4x_wm_values *wm)
934{
935 enum pipe pipe;
936
937 for_each_pipe(dev_priv, pipe)
938 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
939
940 intel_uncore_write(&dev_priv->uncore, DSPFW1,
941 FW_WM(wm->sr.plane, SR) |
942 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
943 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
944 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
945 intel_uncore_write(&dev_priv->uncore, DSPFW2,
946 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
947 FW_WM(wm->sr.fbc, FBC_SR) |
948 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
949 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
950 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
951 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
952 intel_uncore_write(&dev_priv->uncore, DSPFW3,
953 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
954 FW_WM(wm->sr.cursor, CURSOR_SR) |
955 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
956 FW_WM(wm->hpll.plane, HPLL_SR));
957
958 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
959}
960
961#define FW_WM_VLV(value, plane) \
962 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
963
964static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
965 const struct vlv_wm_values *wm)
966{
967 enum pipe pipe;
968
969 for_each_pipe(dev_priv, pipe) {
970 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
971
972 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
973 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
974 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
975 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
976 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
977 }
978
979 /*
980 * Zero the (unused) WM1 watermarks, and also clear all the
981 * high order bits so that there are no out of bounds values
982 * present in the registers during the reprogramming.
983 */
984 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
985 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
986 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
987 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
988 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
989
990 intel_uncore_write(&dev_priv->uncore, DSPFW1,
991 FW_WM(wm->sr.plane, SR) |
992 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
993 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
994 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
995 intel_uncore_write(&dev_priv->uncore, DSPFW2,
996 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
998 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
999 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1000 FW_WM(wm->sr.cursor, CURSOR_SR));
1001
1002 if (IS_CHERRYVIEW(dev_priv)) {
1003 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1004 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1005 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1006 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1007 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1008 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1009 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1010 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1011 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1012 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1013 FW_WM(wm->sr.plane >> 9, SR_HI) |
1014 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1015 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1016 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1017 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1018 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1019 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1020 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1021 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1022 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1023 } else {
1024 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1025 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1026 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1027 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1028 FW_WM(wm->sr.plane >> 9, SR_HI) |
1029 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1030 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1031 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1032 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1033 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1034 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1035 }
1036
1037 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1038}
1039
1040#undef FW_WM_VLV
1041
1042static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1043{
1044 /* all latencies in usec */
1045 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1046 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1047 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1048
1049 dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1050}
1051
1052static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1053{
1054 /*
1055 * DSPCNTR[13] supposedly controls whether the
1056 * primary plane can use the FIFO space otherwise
1057 * reserved for the sprite plane. It's not 100% clear
1058 * what the actual FIFO size is, but it looks like we
1059 * can happily set both primary and sprite watermarks
1060 * up to 127 cachelines. So that would seem to mean
1061 * that either DSPCNTR[13] doesn't do anything, or that
1062 * the total FIFO is >= 256 cachelines in size. Either
1063 * way, we don't seem to have to worry about this
1064 * repartitioning as the maximum watermark value the
1065 * register can hold for each plane is lower than the
1066 * minimum FIFO size.
1067 */
1068 switch (plane_id) {
1069 case PLANE_CURSOR:
1070 return 63;
1071 case PLANE_PRIMARY:
1072 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1073 case PLANE_SPRITE0:
1074 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1075 default:
1076 MISSING_CASE(plane_id);
1077 return 0;
1078 }
1079}
1080
1081static int g4x_fbc_fifo_size(int level)
1082{
1083 switch (level) {
1084 case G4X_WM_LEVEL_SR:
1085 return 7;
1086 case G4X_WM_LEVEL_HPLL:
1087 return 15;
1088 default:
1089 MISSING_CASE(level);
1090 return 0;
1091 }
1092}
1093
1094static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1095 const struct intel_plane_state *plane_state,
1096 int level)
1097{
1098 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1099 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1100 const struct drm_display_mode *pipe_mode =
1101 &crtc_state->hw.pipe_mode;
1102 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1103 unsigned int pixel_rate, htotal, cpp, width, wm;
1104
1105 if (latency == 0)
1106 return USHRT_MAX;
1107
1108 if (!intel_wm_plane_visible(crtc_state, plane_state))
1109 return 0;
1110
1111 cpp = plane_state->hw.fb->format->cpp[0];
1112
1113 /*
1114 * WaUse32BppForSRWM:ctg,elk
1115 *
1116 * The spec fails to list this restriction for the
1117 * HPLL watermark, which seems a little strange.
1118 * Let's use 32bpp for the HPLL watermark as well.
1119 */
1120 if (plane->id == PLANE_PRIMARY &&
1121 level != G4X_WM_LEVEL_NORMAL)
1122 cpp = max(cpp, 4u);
1123
1124 pixel_rate = crtc_state->pixel_rate;
1125 htotal = pipe_mode->crtc_htotal;
1126 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1127
1128 if (plane->id == PLANE_CURSOR) {
1129 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1130 } else if (plane->id == PLANE_PRIMARY &&
1131 level == G4X_WM_LEVEL_NORMAL) {
1132 wm = intel_wm_method1(pixel_rate, cpp, latency);
1133 } else {
1134 unsigned int small, large;
1135
1136 small = intel_wm_method1(pixel_rate, cpp, latency);
1137 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1138
1139 wm = min(small, large);
1140 }
1141
1142 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1143 width, cpp);
1144
1145 wm = DIV_ROUND_UP(wm, 64) + 2;
1146
1147 return min_t(unsigned int, wm, USHRT_MAX);
1148}
1149
1150static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1151 int level, enum plane_id plane_id, u16 value)
1152{
1153 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1154 bool dirty = false;
1155
1156 for (; level < intel_wm_num_levels(dev_priv); level++) {
1157 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1158
1159 dirty |= raw->plane[plane_id] != value;
1160 raw->plane[plane_id] = value;
1161 }
1162
1163 return dirty;
1164}
1165
1166static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1167 int level, u16 value)
1168{
1169 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1170 bool dirty = false;
1171
1172 /* NORMAL level doesn't have an FBC watermark */
1173 level = max(level, G4X_WM_LEVEL_SR);
1174
1175 for (; level < intel_wm_num_levels(dev_priv); level++) {
1176 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1177
1178 dirty |= raw->fbc != value;
1179 raw->fbc = value;
1180 }
1181
1182 return dirty;
1183}
1184
1185static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1186 const struct intel_plane_state *plane_state,
1187 u32 pri_val);
1188
1189static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1190 const struct intel_plane_state *plane_state)
1191{
1192 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1193 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1195 enum plane_id plane_id = plane->id;
1196 bool dirty = false;
1197 int level;
1198
1199 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1200 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1201 if (plane_id == PLANE_PRIMARY)
1202 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1203 goto out;
1204 }
1205
1206 for (level = 0; level < num_levels; level++) {
1207 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1208 int wm, max_wm;
1209
1210 wm = g4x_compute_wm(crtc_state, plane_state, level);
1211 max_wm = g4x_plane_fifo_size(plane_id, level);
1212
1213 if (wm > max_wm)
1214 break;
1215
1216 dirty |= raw->plane[plane_id] != wm;
1217 raw->plane[plane_id] = wm;
1218
1219 if (plane_id != PLANE_PRIMARY ||
1220 level == G4X_WM_LEVEL_NORMAL)
1221 continue;
1222
1223 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1224 raw->plane[plane_id]);
1225 max_wm = g4x_fbc_fifo_size(level);
1226
1227 /*
1228 * FBC wm is not mandatory as we
1229 * can always just disable its use.
1230 */
1231 if (wm > max_wm)
1232 wm = USHRT_MAX;
1233
1234 dirty |= raw->fbc != wm;
1235 raw->fbc = wm;
1236 }
1237
1238 /* mark watermarks as invalid */
1239 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1240
1241 if (plane_id == PLANE_PRIMARY)
1242 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1243
1244 out:
1245 if (dirty) {
1246 drm_dbg_kms(&dev_priv->drm,
1247 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1248 plane->base.name,
1249 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1250 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1251 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1252
1253 if (plane_id == PLANE_PRIMARY)
1254 drm_dbg_kms(&dev_priv->drm,
1255 "FBC watermarks: SR=%d, HPLL=%d\n",
1256 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1257 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1258 }
1259
1260 return dirty;
1261}
1262
1263static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1264 enum plane_id plane_id, int level)
1265{
1266 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1267
1268 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1269}
1270
1271static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1272 int level)
1273{
1274 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1275
1276 if (level > dev_priv->display.wm.max_level)
1277 return false;
1278
1279 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1280 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1281 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1282}
1283
1284/* mark all levels starting from 'level' as invalid */
1285static void g4x_invalidate_wms(struct intel_crtc *crtc,
1286 struct g4x_wm_state *wm_state, int level)
1287{
1288 if (level <= G4X_WM_LEVEL_NORMAL) {
1289 enum plane_id plane_id;
1290
1291 for_each_plane_id_on_crtc(crtc, plane_id)
1292 wm_state->wm.plane[plane_id] = USHRT_MAX;
1293 }
1294
1295 if (level <= G4X_WM_LEVEL_SR) {
1296 wm_state->cxsr = false;
1297 wm_state->sr.cursor = USHRT_MAX;
1298 wm_state->sr.plane = USHRT_MAX;
1299 wm_state->sr.fbc = USHRT_MAX;
1300 }
1301
1302 if (level <= G4X_WM_LEVEL_HPLL) {
1303 wm_state->hpll_en = false;
1304 wm_state->hpll.cursor = USHRT_MAX;
1305 wm_state->hpll.plane = USHRT_MAX;
1306 wm_state->hpll.fbc = USHRT_MAX;
1307 }
1308}
1309
1310static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1311 int level)
1312{
1313 if (level < G4X_WM_LEVEL_SR)
1314 return false;
1315
1316 if (level >= G4X_WM_LEVEL_SR &&
1317 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1318 return false;
1319
1320 if (level >= G4X_WM_LEVEL_HPLL &&
1321 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1322 return false;
1323
1324 return true;
1325}
1326
1327static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1328{
1329 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1330 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1331 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1332 const struct g4x_pipe_wm *raw;
1333 enum plane_id plane_id;
1334 int level;
1335
1336 level = G4X_WM_LEVEL_NORMAL;
1337 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1338 goto out;
1339
1340 raw = &crtc_state->wm.g4x.raw[level];
1341 for_each_plane_id_on_crtc(crtc, plane_id)
1342 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1343
1344 level = G4X_WM_LEVEL_SR;
1345 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1346 goto out;
1347
1348 raw = &crtc_state->wm.g4x.raw[level];
1349 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1350 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1351 wm_state->sr.fbc = raw->fbc;
1352
1353 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1354
1355 level = G4X_WM_LEVEL_HPLL;
1356 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1357 goto out;
1358
1359 raw = &crtc_state->wm.g4x.raw[level];
1360 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1361 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1362 wm_state->hpll.fbc = raw->fbc;
1363
1364 wm_state->hpll_en = wm_state->cxsr;
1365
1366 level++;
1367
1368 out:
1369 if (level == G4X_WM_LEVEL_NORMAL)
1370 return -EINVAL;
1371
1372 /* invalidate the higher levels */
1373 g4x_invalidate_wms(crtc, wm_state, level);
1374
1375 /*
1376 * Determine if the FBC watermark(s) can be used. IF
1377 * this isn't the case we prefer to disable the FBC
1378 * watermark(s) rather than disable the SR/HPLL
1379 * level(s) entirely. 'level-1' is the highest valid
1380 * level here.
1381 */
1382 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1383
1384 return 0;
1385}
1386
1387static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1388 struct intel_crtc *crtc)
1389{
1390 struct intel_crtc_state *crtc_state =
1391 intel_atomic_get_new_crtc_state(state, crtc);
1392 const struct intel_plane_state *old_plane_state;
1393 const struct intel_plane_state *new_plane_state;
1394 struct intel_plane *plane;
1395 unsigned int dirty = 0;
1396 int i;
1397
1398 for_each_oldnew_intel_plane_in_state(state, plane,
1399 old_plane_state,
1400 new_plane_state, i) {
1401 if (new_plane_state->hw.crtc != &crtc->base &&
1402 old_plane_state->hw.crtc != &crtc->base)
1403 continue;
1404
1405 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1406 dirty |= BIT(plane->id);
1407 }
1408
1409 if (!dirty)
1410 return 0;
1411
1412 return _g4x_compute_pipe_wm(crtc_state);
1413}
1414
1415static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1416 struct intel_crtc *crtc)
1417{
1418 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1419 struct intel_crtc_state *new_crtc_state =
1420 intel_atomic_get_new_crtc_state(state, crtc);
1421 const struct intel_crtc_state *old_crtc_state =
1422 intel_atomic_get_old_crtc_state(state, crtc);
1423 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1424 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1425 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1426 enum plane_id plane_id;
1427
1428 if (!new_crtc_state->hw.active ||
1429 intel_crtc_needs_modeset(new_crtc_state)) {
1430 *intermediate = *optimal;
1431
1432 intermediate->cxsr = false;
1433 intermediate->hpll_en = false;
1434 goto out;
1435 }
1436
1437 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1438 !new_crtc_state->disable_cxsr;
1439 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1440 !new_crtc_state->disable_cxsr;
1441 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1442
1443 for_each_plane_id_on_crtc(crtc, plane_id) {
1444 intermediate->wm.plane[plane_id] =
1445 max(optimal->wm.plane[plane_id],
1446 active->wm.plane[plane_id]);
1447
1448 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1449 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1450 }
1451
1452 intermediate->sr.plane = max(optimal->sr.plane,
1453 active->sr.plane);
1454 intermediate->sr.cursor = max(optimal->sr.cursor,
1455 active->sr.cursor);
1456 intermediate->sr.fbc = max(optimal->sr.fbc,
1457 active->sr.fbc);
1458
1459 intermediate->hpll.plane = max(optimal->hpll.plane,
1460 active->hpll.plane);
1461 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1462 active->hpll.cursor);
1463 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1464 active->hpll.fbc);
1465
1466 drm_WARN_ON(&dev_priv->drm,
1467 (intermediate->sr.plane >
1468 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1469 intermediate->sr.cursor >
1470 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1471 intermediate->cxsr);
1472 drm_WARN_ON(&dev_priv->drm,
1473 (intermediate->sr.plane >
1474 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1475 intermediate->sr.cursor >
1476 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1477 intermediate->hpll_en);
1478
1479 drm_WARN_ON(&dev_priv->drm,
1480 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1481 intermediate->fbc_en && intermediate->cxsr);
1482 drm_WARN_ON(&dev_priv->drm,
1483 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1484 intermediate->fbc_en && intermediate->hpll_en);
1485
1486out:
1487 /*
1488 * If our intermediate WM are identical to the final WM, then we can
1489 * omit the post-vblank programming; only update if it's different.
1490 */
1491 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1492 new_crtc_state->wm.need_postvbl_update = true;
1493
1494 return 0;
1495}
1496
1497static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1498 struct g4x_wm_values *wm)
1499{
1500 struct intel_crtc *crtc;
1501 int num_active_pipes = 0;
1502
1503 wm->cxsr = true;
1504 wm->hpll_en = true;
1505 wm->fbc_en = true;
1506
1507 for_each_intel_crtc(&dev_priv->drm, crtc) {
1508 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1509
1510 if (!crtc->active)
1511 continue;
1512
1513 if (!wm_state->cxsr)
1514 wm->cxsr = false;
1515 if (!wm_state->hpll_en)
1516 wm->hpll_en = false;
1517 if (!wm_state->fbc_en)
1518 wm->fbc_en = false;
1519
1520 num_active_pipes++;
1521 }
1522
1523 if (num_active_pipes != 1) {
1524 wm->cxsr = false;
1525 wm->hpll_en = false;
1526 wm->fbc_en = false;
1527 }
1528
1529 for_each_intel_crtc(&dev_priv->drm, crtc) {
1530 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1531 enum pipe pipe = crtc->pipe;
1532
1533 wm->pipe[pipe] = wm_state->wm;
1534 if (crtc->active && wm->cxsr)
1535 wm->sr = wm_state->sr;
1536 if (crtc->active && wm->hpll_en)
1537 wm->hpll = wm_state->hpll;
1538 }
1539}
1540
1541static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1542{
1543 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1544 struct g4x_wm_values new_wm = {};
1545
1546 g4x_merge_wm(dev_priv, &new_wm);
1547
1548 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1549 return;
1550
1551 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1552 _intel_set_memory_cxsr(dev_priv, false);
1553
1554 g4x_write_wm_values(dev_priv, &new_wm);
1555
1556 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1557 _intel_set_memory_cxsr(dev_priv, true);
1558
1559 *old_wm = new_wm;
1560}
1561
1562static void g4x_initial_watermarks(struct intel_atomic_state *state,
1563 struct intel_crtc *crtc)
1564{
1565 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1566 const struct intel_crtc_state *crtc_state =
1567 intel_atomic_get_new_crtc_state(state, crtc);
1568
1569 mutex_lock(&dev_priv->display.wm.wm_mutex);
1570 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1571 g4x_program_watermarks(dev_priv);
1572 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1573}
1574
1575static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1576 struct intel_crtc *crtc)
1577{
1578 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1579 const struct intel_crtc_state *crtc_state =
1580 intel_atomic_get_new_crtc_state(state, crtc);
1581
1582 if (!crtc_state->wm.need_postvbl_update)
1583 return;
1584
1585 mutex_lock(&dev_priv->display.wm.wm_mutex);
1586 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1587 g4x_program_watermarks(dev_priv);
1588 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1589}
1590
1591/* latency must be in 0.1us units. */
1592static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1593 unsigned int htotal,
1594 unsigned int width,
1595 unsigned int cpp,
1596 unsigned int latency)
1597{
1598 unsigned int ret;
1599
1600 ret = intel_wm_method2(pixel_rate, htotal,
1601 width, cpp, latency);
1602 ret = DIV_ROUND_UP(ret, 64);
1603
1604 return ret;
1605}
1606
1607static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1608{
1609 /* all latencies in usec */
1610 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1611
1612 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1613
1614 if (IS_CHERRYVIEW(dev_priv)) {
1615 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1616 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1617
1618 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1619 }
1620}
1621
1622static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1623 const struct intel_plane_state *plane_state,
1624 int level)
1625{
1626 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1627 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1628 const struct drm_display_mode *pipe_mode =
1629 &crtc_state->hw.pipe_mode;
1630 unsigned int pixel_rate, htotal, cpp, width, wm;
1631
1632 if (dev_priv->display.wm.pri_latency[level] == 0)
1633 return USHRT_MAX;
1634
1635 if (!intel_wm_plane_visible(crtc_state, plane_state))
1636 return 0;
1637
1638 cpp = plane_state->hw.fb->format->cpp[0];
1639 pixel_rate = crtc_state->pixel_rate;
1640 htotal = pipe_mode->crtc_htotal;
1641 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1642
1643 if (plane->id == PLANE_CURSOR) {
1644 /*
1645 * FIXME the formula gives values that are
1646 * too big for the cursor FIFO, and hence we
1647 * would never be able to use cursors. For
1648 * now just hardcode the watermark.
1649 */
1650 wm = 63;
1651 } else {
1652 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1653 dev_priv->display.wm.pri_latency[level] * 10);
1654 }
1655
1656 return min_t(unsigned int, wm, USHRT_MAX);
1657}
1658
1659static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1660{
1661 return (active_planes & (BIT(PLANE_SPRITE0) |
1662 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1663}
1664
1665static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1666{
1667 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1668 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1669 const struct g4x_pipe_wm *raw =
1670 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1671 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1672 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1673 int num_active_planes = hweight8(active_planes);
1674 const int fifo_size = 511;
1675 int fifo_extra, fifo_left = fifo_size;
1676 int sprite0_fifo_extra = 0;
1677 unsigned int total_rate;
1678 enum plane_id plane_id;
1679
1680 /*
1681 * When enabling sprite0 after sprite1 has already been enabled
1682 * we tend to get an underrun unless sprite0 already has some
1683 * FIFO space allcoated. Hence we always allocate at least one
1684 * cacheline for sprite0 whenever sprite1 is enabled.
1685 *
1686 * All other plane enable sequences appear immune to this problem.
1687 */
1688 if (vlv_need_sprite0_fifo_workaround(active_planes))
1689 sprite0_fifo_extra = 1;
1690
1691 total_rate = raw->plane[PLANE_PRIMARY] +
1692 raw->plane[PLANE_SPRITE0] +
1693 raw->plane[PLANE_SPRITE1] +
1694 sprite0_fifo_extra;
1695
1696 if (total_rate > fifo_size)
1697 return -EINVAL;
1698
1699 if (total_rate == 0)
1700 total_rate = 1;
1701
1702 for_each_plane_id_on_crtc(crtc, plane_id) {
1703 unsigned int rate;
1704
1705 if ((active_planes & BIT(plane_id)) == 0) {
1706 fifo_state->plane[plane_id] = 0;
1707 continue;
1708 }
1709
1710 rate = raw->plane[plane_id];
1711 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1712 fifo_left -= fifo_state->plane[plane_id];
1713 }
1714
1715 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1716 fifo_left -= sprite0_fifo_extra;
1717
1718 fifo_state->plane[PLANE_CURSOR] = 63;
1719
1720 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1721
1722 /* spread the remainder evenly */
1723 for_each_plane_id_on_crtc(crtc, plane_id) {
1724 int plane_extra;
1725
1726 if (fifo_left == 0)
1727 break;
1728
1729 if ((active_planes & BIT(plane_id)) == 0)
1730 continue;
1731
1732 plane_extra = min(fifo_extra, fifo_left);
1733 fifo_state->plane[plane_id] += plane_extra;
1734 fifo_left -= plane_extra;
1735 }
1736
1737 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1738
1739 /* give it all to the first plane if none are active */
1740 if (active_planes == 0) {
1741 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1742 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1743 }
1744
1745 return 0;
1746}
1747
1748/* mark all levels starting from 'level' as invalid */
1749static void vlv_invalidate_wms(struct intel_crtc *crtc,
1750 struct vlv_wm_state *wm_state, int level)
1751{
1752 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1753
1754 for (; level < intel_wm_num_levels(dev_priv); level++) {
1755 enum plane_id plane_id;
1756
1757 for_each_plane_id_on_crtc(crtc, plane_id)
1758 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1759
1760 wm_state->sr[level].cursor = USHRT_MAX;
1761 wm_state->sr[level].plane = USHRT_MAX;
1762 }
1763}
1764
1765static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1766{
1767 if (wm > fifo_size)
1768 return USHRT_MAX;
1769 else
1770 return fifo_size - wm;
1771}
1772
1773/*
1774 * Starting from 'level' set all higher
1775 * levels to 'value' in the "raw" watermarks.
1776 */
1777static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1778 int level, enum plane_id plane_id, u16 value)
1779{
1780 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1781 int num_levels = intel_wm_num_levels(dev_priv);
1782 bool dirty = false;
1783
1784 for (; level < num_levels; level++) {
1785 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1786
1787 dirty |= raw->plane[plane_id] != value;
1788 raw->plane[plane_id] = value;
1789 }
1790
1791 return dirty;
1792}
1793
1794static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1795 const struct intel_plane_state *plane_state)
1796{
1797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1798 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799 enum plane_id plane_id = plane->id;
1800 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801 int level;
1802 bool dirty = false;
1803
1804 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1805 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806 goto out;
1807 }
1808
1809 for (level = 0; level < num_levels; level++) {
1810 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1811 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1812 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1813
1814 if (wm > max_wm)
1815 break;
1816
1817 dirty |= raw->plane[plane_id] != wm;
1818 raw->plane[plane_id] = wm;
1819 }
1820
1821 /* mark all higher levels as invalid */
1822 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823
1824out:
1825 if (dirty)
1826 drm_dbg_kms(&dev_priv->drm,
1827 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1828 plane->base.name,
1829 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1830 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1831 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1832
1833 return dirty;
1834}
1835
1836static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1837 enum plane_id plane_id, int level)
1838{
1839 const struct g4x_pipe_wm *raw =
1840 &crtc_state->wm.vlv.raw[level];
1841 const struct vlv_fifo_state *fifo_state =
1842 &crtc_state->wm.vlv.fifo_state;
1843
1844 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1845}
1846
1847static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1848{
1849 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1850 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1851 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1852 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1853}
1854
1855static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1856{
1857 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1858 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1860 const struct vlv_fifo_state *fifo_state =
1861 &crtc_state->wm.vlv.fifo_state;
1862 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1863 int num_active_planes = hweight8(active_planes);
1864 enum plane_id plane_id;
1865 int level;
1866
1867 /* initially allow all levels */
1868 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1869 /*
1870 * Note that enabling cxsr with no primary/sprite planes
1871 * enabled can wedge the pipe. Hence we only allow cxsr
1872 * with exactly one enabled primary/sprite plane.
1873 */
1874 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1875
1876 for (level = 0; level < wm_state->num_levels; level++) {
1877 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1878 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1879
1880 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1881 break;
1882
1883 for_each_plane_id_on_crtc(crtc, plane_id) {
1884 wm_state->wm[level].plane[plane_id] =
1885 vlv_invert_wm_value(raw->plane[plane_id],
1886 fifo_state->plane[plane_id]);
1887 }
1888
1889 wm_state->sr[level].plane =
1890 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1891 raw->plane[PLANE_SPRITE0],
1892 raw->plane[PLANE_SPRITE1]),
1893 sr_fifo_size);
1894
1895 wm_state->sr[level].cursor =
1896 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1897 63);
1898 }
1899
1900 if (level == 0)
1901 return -EINVAL;
1902
1903 /* limit to only levels we can actually handle */
1904 wm_state->num_levels = level;
1905
1906 /* invalidate the higher levels */
1907 vlv_invalidate_wms(crtc, wm_state, level);
1908
1909 return 0;
1910}
1911
1912static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1913 struct intel_crtc *crtc)
1914{
1915 struct intel_crtc_state *crtc_state =
1916 intel_atomic_get_new_crtc_state(state, crtc);
1917 const struct intel_plane_state *old_plane_state;
1918 const struct intel_plane_state *new_plane_state;
1919 struct intel_plane *plane;
1920 unsigned int dirty = 0;
1921 int i;
1922
1923 for_each_oldnew_intel_plane_in_state(state, plane,
1924 old_plane_state,
1925 new_plane_state, i) {
1926 if (new_plane_state->hw.crtc != &crtc->base &&
1927 old_plane_state->hw.crtc != &crtc->base)
1928 continue;
1929
1930 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1931 dirty |= BIT(plane->id);
1932 }
1933
1934 /*
1935 * DSPARB registers may have been reset due to the
1936 * power well being turned off. Make sure we restore
1937 * them to a consistent state even if no primary/sprite
1938 * planes are initially active. We also force a FIFO
1939 * recomputation so that we are sure to sanitize the
1940 * FIFO setting we took over from the BIOS even if there
1941 * are no active planes on the crtc.
1942 */
1943 if (intel_crtc_needs_modeset(crtc_state))
1944 dirty = ~0;
1945
1946 if (!dirty)
1947 return 0;
1948
1949 /* cursor changes don't warrant a FIFO recompute */
1950 if (dirty & ~BIT(PLANE_CURSOR)) {
1951 const struct intel_crtc_state *old_crtc_state =
1952 intel_atomic_get_old_crtc_state(state, crtc);
1953 const struct vlv_fifo_state *old_fifo_state =
1954 &old_crtc_state->wm.vlv.fifo_state;
1955 const struct vlv_fifo_state *new_fifo_state =
1956 &crtc_state->wm.vlv.fifo_state;
1957 int ret;
1958
1959 ret = vlv_compute_fifo(crtc_state);
1960 if (ret)
1961 return ret;
1962
1963 if (intel_crtc_needs_modeset(crtc_state) ||
1964 memcmp(old_fifo_state, new_fifo_state,
1965 sizeof(*new_fifo_state)) != 0)
1966 crtc_state->fifo_changed = true;
1967 }
1968
1969 return _vlv_compute_pipe_wm(crtc_state);
1970}
1971
1972#define VLV_FIFO(plane, value) \
1973 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1974
1975static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1976 struct intel_crtc *crtc)
1977{
1978 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1979 struct intel_uncore *uncore = &dev_priv->uncore;
1980 const struct intel_crtc_state *crtc_state =
1981 intel_atomic_get_new_crtc_state(state, crtc);
1982 const struct vlv_fifo_state *fifo_state =
1983 &crtc_state->wm.vlv.fifo_state;
1984 int sprite0_start, sprite1_start, fifo_size;
1985 u32 dsparb, dsparb2, dsparb3;
1986
1987 if (!crtc_state->fifo_changed)
1988 return;
1989
1990 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1991 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1992 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1993
1994 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1995 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1996
1997 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1998
1999 /*
2000 * uncore.lock serves a double purpose here. It allows us to
2001 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2002 * it protects the DSPARB registers from getting clobbered by
2003 * parallel updates from multiple pipes.
2004 *
2005 * intel_pipe_update_start() has already disabled interrupts
2006 * for us, so a plain spin_lock() is sufficient here.
2007 */
2008 spin_lock(&uncore->lock);
2009
2010 switch (crtc->pipe) {
2011 case PIPE_A:
2012 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2013 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2014
2015 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2016 VLV_FIFO(SPRITEB, 0xff));
2017 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2018 VLV_FIFO(SPRITEB, sprite1_start));
2019
2020 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2021 VLV_FIFO(SPRITEB_HI, 0x1));
2022 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2023 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2024
2025 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2026 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2027 break;
2028 case PIPE_B:
2029 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2030 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2031
2032 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2033 VLV_FIFO(SPRITED, 0xff));
2034 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2035 VLV_FIFO(SPRITED, sprite1_start));
2036
2037 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2038 VLV_FIFO(SPRITED_HI, 0xff));
2039 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2040 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2041
2042 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2043 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2044 break;
2045 case PIPE_C:
2046 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2047 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2048
2049 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2050 VLV_FIFO(SPRITEF, 0xff));
2051 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2052 VLV_FIFO(SPRITEF, sprite1_start));
2053
2054 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2055 VLV_FIFO(SPRITEF_HI, 0xff));
2056 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2057 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2058
2059 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2060 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2061 break;
2062 default:
2063 break;
2064 }
2065
2066 intel_uncore_posting_read_fw(uncore, DSPARB);
2067
2068 spin_unlock(&uncore->lock);
2069}
2070
2071#undef VLV_FIFO
2072
2073static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2074 struct intel_crtc *crtc)
2075{
2076 struct intel_crtc_state *new_crtc_state =
2077 intel_atomic_get_new_crtc_state(state, crtc);
2078 const struct intel_crtc_state *old_crtc_state =
2079 intel_atomic_get_old_crtc_state(state, crtc);
2080 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2081 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2082 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2083 int level;
2084
2085 if (!new_crtc_state->hw.active ||
2086 intel_crtc_needs_modeset(new_crtc_state)) {
2087 *intermediate = *optimal;
2088
2089 intermediate->cxsr = false;
2090 goto out;
2091 }
2092
2093 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2094 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2095 !new_crtc_state->disable_cxsr;
2096
2097 for (level = 0; level < intermediate->num_levels; level++) {
2098 enum plane_id plane_id;
2099
2100 for_each_plane_id_on_crtc(crtc, plane_id) {
2101 intermediate->wm[level].plane[plane_id] =
2102 min(optimal->wm[level].plane[plane_id],
2103 active->wm[level].plane[plane_id]);
2104 }
2105
2106 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2107 active->sr[level].plane);
2108 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2109 active->sr[level].cursor);
2110 }
2111
2112 vlv_invalidate_wms(crtc, intermediate, level);
2113
2114out:
2115 /*
2116 * If our intermediate WM are identical to the final WM, then we can
2117 * omit the post-vblank programming; only update if it's different.
2118 */
2119 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2120 new_crtc_state->wm.need_postvbl_update = true;
2121
2122 return 0;
2123}
2124
2125static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2126 struct vlv_wm_values *wm)
2127{
2128 struct intel_crtc *crtc;
2129 int num_active_pipes = 0;
2130
2131 wm->level = dev_priv->display.wm.max_level;
2132 wm->cxsr = true;
2133
2134 for_each_intel_crtc(&dev_priv->drm, crtc) {
2135 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2136
2137 if (!crtc->active)
2138 continue;
2139
2140 if (!wm_state->cxsr)
2141 wm->cxsr = false;
2142
2143 num_active_pipes++;
2144 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2145 }
2146
2147 if (num_active_pipes != 1)
2148 wm->cxsr = false;
2149
2150 if (num_active_pipes > 1)
2151 wm->level = VLV_WM_LEVEL_PM2;
2152
2153 for_each_intel_crtc(&dev_priv->drm, crtc) {
2154 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2155 enum pipe pipe = crtc->pipe;
2156
2157 wm->pipe[pipe] = wm_state->wm[wm->level];
2158 if (crtc->active && wm->cxsr)
2159 wm->sr = wm_state->sr[wm->level];
2160
2161 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2162 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2163 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2164 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2165 }
2166}
2167
2168static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2169{
2170 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2171 struct vlv_wm_values new_wm = {};
2172
2173 vlv_merge_wm(dev_priv, &new_wm);
2174
2175 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2176 return;
2177
2178 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2179 chv_set_memory_dvfs(dev_priv, false);
2180
2181 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2182 chv_set_memory_pm5(dev_priv, false);
2183
2184 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2185 _intel_set_memory_cxsr(dev_priv, false);
2186
2187 vlv_write_wm_values(dev_priv, &new_wm);
2188
2189 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2190 _intel_set_memory_cxsr(dev_priv, true);
2191
2192 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2193 chv_set_memory_pm5(dev_priv, true);
2194
2195 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2196 chv_set_memory_dvfs(dev_priv, true);
2197
2198 *old_wm = new_wm;
2199}
2200
2201static void vlv_initial_watermarks(struct intel_atomic_state *state,
2202 struct intel_crtc *crtc)
2203{
2204 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2205 const struct intel_crtc_state *crtc_state =
2206 intel_atomic_get_new_crtc_state(state, crtc);
2207
2208 mutex_lock(&dev_priv->display.wm.wm_mutex);
2209 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2210 vlv_program_watermarks(dev_priv);
2211 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2212}
2213
2214static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2215 struct intel_crtc *crtc)
2216{
2217 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2218 const struct intel_crtc_state *crtc_state =
2219 intel_atomic_get_new_crtc_state(state, crtc);
2220
2221 if (!crtc_state->wm.need_postvbl_update)
2222 return;
2223
2224 mutex_lock(&dev_priv->display.wm.wm_mutex);
2225 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2226 vlv_program_watermarks(dev_priv);
2227 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2228}
2229
2230static void i965_update_wm(struct drm_i915_private *dev_priv)
2231{
2232 struct intel_crtc *crtc;
2233 int srwm = 1;
2234 int cursor_sr = 16;
2235 bool cxsr_enabled;
2236
2237 /* Calc sr entries for one plane configs */
2238 crtc = single_enabled_crtc(dev_priv);
2239 if (crtc) {
2240 /* self-refresh has much higher latency */
2241 static const int sr_latency_ns = 12000;
2242 const struct drm_display_mode *pipe_mode =
2243 &crtc->config->hw.pipe_mode;
2244 const struct drm_framebuffer *fb =
2245 crtc->base.primary->state->fb;
2246 int pixel_rate = crtc->config->pixel_rate;
2247 int htotal = pipe_mode->crtc_htotal;
2248 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2249 int cpp = fb->format->cpp[0];
2250 int entries;
2251
2252 entries = intel_wm_method2(pixel_rate, htotal,
2253 width, cpp, sr_latency_ns / 100);
2254 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2255 srwm = I965_FIFO_SIZE - entries;
2256 if (srwm < 0)
2257 srwm = 1;
2258 srwm &= 0x1ff;
2259 drm_dbg_kms(&dev_priv->drm,
2260 "self-refresh entries: %d, wm: %d\n",
2261 entries, srwm);
2262
2263 entries = intel_wm_method2(pixel_rate, htotal,
2264 crtc->base.cursor->state->crtc_w, 4,
2265 sr_latency_ns / 100);
2266 entries = DIV_ROUND_UP(entries,
2267 i965_cursor_wm_info.cacheline_size) +
2268 i965_cursor_wm_info.guard_size;
2269
2270 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2271 if (cursor_sr > i965_cursor_wm_info.max_wm)
2272 cursor_sr = i965_cursor_wm_info.max_wm;
2273
2274 drm_dbg_kms(&dev_priv->drm,
2275 "self-refresh watermark: display plane %d "
2276 "cursor %d\n", srwm, cursor_sr);
2277
2278 cxsr_enabled = true;
2279 } else {
2280 cxsr_enabled = false;
2281 /* Turn off self refresh if both pipes are enabled */
2282 intel_set_memory_cxsr(dev_priv, false);
2283 }
2284
2285 drm_dbg_kms(&dev_priv->drm,
2286 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2287 srwm);
2288
2289 /* 965 has limitations... */
2290 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2291 FW_WM(8, CURSORB) |
2292 FW_WM(8, PLANEB) |
2293 FW_WM(8, PLANEA));
2294 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2295 FW_WM(8, PLANEC_OLD));
2296 /* update cursor SR watermark */
2297 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2298
2299 if (cxsr_enabled)
2300 intel_set_memory_cxsr(dev_priv, true);
2301}
2302
2303#undef FW_WM
2304
2305static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2306 enum i9xx_plane_id i9xx_plane)
2307{
2308 struct intel_plane *plane;
2309
2310 for_each_intel_plane(&i915->drm, plane) {
2311 if (plane->id == PLANE_PRIMARY &&
2312 plane->i9xx_plane == i9xx_plane)
2313 return intel_crtc_for_pipe(i915, plane->pipe);
2314 }
2315
2316 return NULL;
2317}
2318
2319static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2320{
2321 const struct intel_watermark_params *wm_info;
2322 u32 fwater_lo;
2323 u32 fwater_hi;
2324 int cwm, srwm = 1;
2325 int fifo_size;
2326 int planea_wm, planeb_wm;
2327 struct intel_crtc *crtc;
2328
2329 if (IS_I945GM(dev_priv))
2330 wm_info = &i945_wm_info;
2331 else if (DISPLAY_VER(dev_priv) != 2)
2332 wm_info = &i915_wm_info;
2333 else
2334 wm_info = &i830_a_wm_info;
2335
2336 if (DISPLAY_VER(dev_priv) == 2)
2337 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2338 else
2339 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2340 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2341 if (intel_crtc_active(crtc)) {
2342 const struct drm_framebuffer *fb =
2343 crtc->base.primary->state->fb;
2344 int cpp;
2345
2346 if (DISPLAY_VER(dev_priv) == 2)
2347 cpp = 4;
2348 else
2349 cpp = fb->format->cpp[0];
2350
2351 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2352 wm_info, fifo_size, cpp,
2353 pessimal_latency_ns);
2354 } else {
2355 planea_wm = fifo_size - wm_info->guard_size;
2356 if (planea_wm > (long)wm_info->max_wm)
2357 planea_wm = wm_info->max_wm;
2358 }
2359
2360 if (DISPLAY_VER(dev_priv) == 2)
2361 wm_info = &i830_bc_wm_info;
2362
2363 if (DISPLAY_VER(dev_priv) == 2)
2364 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2365 else
2366 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2367 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2368 if (intel_crtc_active(crtc)) {
2369 const struct drm_framebuffer *fb =
2370 crtc->base.primary->state->fb;
2371 int cpp;
2372
2373 if (DISPLAY_VER(dev_priv) == 2)
2374 cpp = 4;
2375 else
2376 cpp = fb->format->cpp[0];
2377
2378 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379 wm_info, fifo_size, cpp,
2380 pessimal_latency_ns);
2381 } else {
2382 planeb_wm = fifo_size - wm_info->guard_size;
2383 if (planeb_wm > (long)wm_info->max_wm)
2384 planeb_wm = wm_info->max_wm;
2385 }
2386
2387 drm_dbg_kms(&dev_priv->drm,
2388 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2389
2390 crtc = single_enabled_crtc(dev_priv);
2391 if (IS_I915GM(dev_priv) && crtc) {
2392 struct drm_i915_gem_object *obj;
2393
2394 obj = intel_fb_obj(crtc->base.primary->state->fb);
2395
2396 /* self-refresh seems busted with untiled */
2397 if (!i915_gem_object_is_tiled(obj))
2398 crtc = NULL;
2399 }
2400
2401 /*
2402 * Overlay gets an aggressive default since video jitter is bad.
2403 */
2404 cwm = 2;
2405
2406 /* Play safe and disable self-refresh before adjusting watermarks. */
2407 intel_set_memory_cxsr(dev_priv, false);
2408
2409 /* Calc sr entries for one plane configs */
2410 if (HAS_FW_BLC(dev_priv) && crtc) {
2411 /* self-refresh has much higher latency */
2412 static const int sr_latency_ns = 6000;
2413 const struct drm_display_mode *pipe_mode =
2414 &crtc->config->hw.pipe_mode;
2415 const struct drm_framebuffer *fb =
2416 crtc->base.primary->state->fb;
2417 int pixel_rate = crtc->config->pixel_rate;
2418 int htotal = pipe_mode->crtc_htotal;
2419 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2420 int cpp;
2421 int entries;
2422
2423 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2424 cpp = 4;
2425 else
2426 cpp = fb->format->cpp[0];
2427
2428 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2429 sr_latency_ns / 100);
2430 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2431 drm_dbg_kms(&dev_priv->drm,
2432 "self-refresh entries: %d\n", entries);
2433 srwm = wm_info->fifo_size - entries;
2434 if (srwm < 0)
2435 srwm = 1;
2436
2437 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2438 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2439 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2440 else
2441 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2442 }
2443
2444 drm_dbg_kms(&dev_priv->drm,
2445 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2446 planea_wm, planeb_wm, cwm, srwm);
2447
2448 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2449 fwater_hi = (cwm & 0x1f);
2450
2451 /* Set request length to 8 cachelines per fetch */
2452 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2453 fwater_hi = fwater_hi | (1 << 8);
2454
2455 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2456 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2457
2458 if (crtc)
2459 intel_set_memory_cxsr(dev_priv, true);
2460}
2461
2462static void i845_update_wm(struct drm_i915_private *dev_priv)
2463{
2464 struct intel_crtc *crtc;
2465 u32 fwater_lo;
2466 int planea_wm;
2467
2468 crtc = single_enabled_crtc(dev_priv);
2469 if (crtc == NULL)
2470 return;
2471
2472 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2473 &i845_wm_info,
2474 i845_get_fifo_size(dev_priv, PLANE_A),
2475 4, pessimal_latency_ns);
2476 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2477 fwater_lo |= (3<<8) | planea_wm;
2478
2479 drm_dbg_kms(&dev_priv->drm,
2480 "Setting FIFO watermarks - A: %d\n", planea_wm);
2481
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483}
2484
2485/* latency must be in 0.1us units. */
2486static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2487 unsigned int cpp,
2488 unsigned int latency)
2489{
2490 unsigned int ret;
2491
2492 ret = intel_wm_method1(pixel_rate, cpp, latency);
2493 ret = DIV_ROUND_UP(ret, 64) + 2;
2494
2495 return ret;
2496}
2497
2498/* latency must be in 0.1us units. */
2499static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2500 unsigned int htotal,
2501 unsigned int width,
2502 unsigned int cpp,
2503 unsigned int latency)
2504{
2505 unsigned int ret;
2506
2507 ret = intel_wm_method2(pixel_rate, htotal,
2508 width, cpp, latency);
2509 ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511 return ret;
2512}
2513
2514static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2515{
2516 /*
2517 * Neither of these should be possible since this function shouldn't be
2518 * called if the CRTC is off or the plane is invisible. But let's be
2519 * extra paranoid to avoid a potential divide-by-zero if we screw up
2520 * elsewhere in the driver.
2521 */
2522 if (WARN_ON(!cpp))
2523 return 0;
2524 if (WARN_ON(!horiz_pixels))
2525 return 0;
2526
2527 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2528}
2529
2530struct ilk_wm_maximums {
2531 u16 pri;
2532 u16 spr;
2533 u16 cur;
2534 u16 fbc;
2535};
2536
2537/*
2538 * For both WM_PIPE and WM_LP.
2539 * mem_value must be in 0.1us units.
2540 */
2541static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2542 const struct intel_plane_state *plane_state,
2543 u32 mem_value, bool is_lp)
2544{
2545 u32 method1, method2;
2546 int cpp;
2547
2548 if (mem_value == 0)
2549 return U32_MAX;
2550
2551 if (!intel_wm_plane_visible(crtc_state, plane_state))
2552 return 0;
2553
2554 cpp = plane_state->hw.fb->format->cpp[0];
2555
2556 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2557
2558 if (!is_lp)
2559 return method1;
2560
2561 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2562 crtc_state->hw.pipe_mode.crtc_htotal,
2563 drm_rect_width(&plane_state->uapi.src) >> 16,
2564 cpp, mem_value);
2565
2566 return min(method1, method2);
2567}
2568
2569/*
2570 * For both WM_PIPE and WM_LP.
2571 * mem_value must be in 0.1us units.
2572 */
2573static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2574 const struct intel_plane_state *plane_state,
2575 u32 mem_value)
2576{
2577 u32 method1, method2;
2578 int cpp;
2579
2580 if (mem_value == 0)
2581 return U32_MAX;
2582
2583 if (!intel_wm_plane_visible(crtc_state, plane_state))
2584 return 0;
2585
2586 cpp = plane_state->hw.fb->format->cpp[0];
2587
2588 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2589 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2590 crtc_state->hw.pipe_mode.crtc_htotal,
2591 drm_rect_width(&plane_state->uapi.src) >> 16,
2592 cpp, mem_value);
2593 return min(method1, method2);
2594}
2595
2596/*
2597 * For both WM_PIPE and WM_LP.
2598 * mem_value must be in 0.1us units.
2599 */
2600static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2601 const struct intel_plane_state *plane_state,
2602 u32 mem_value)
2603{
2604 int cpp;
2605
2606 if (mem_value == 0)
2607 return U32_MAX;
2608
2609 if (!intel_wm_plane_visible(crtc_state, plane_state))
2610 return 0;
2611
2612 cpp = plane_state->hw.fb->format->cpp[0];
2613
2614 return ilk_wm_method2(crtc_state->pixel_rate,
2615 crtc_state->hw.pipe_mode.crtc_htotal,
2616 drm_rect_width(&plane_state->uapi.src) >> 16,
2617 cpp, mem_value);
2618}
2619
2620/* Only for WM_LP. */
2621static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2622 const struct intel_plane_state *plane_state,
2623 u32 pri_val)
2624{
2625 int cpp;
2626
2627 if (!intel_wm_plane_visible(crtc_state, plane_state))
2628 return 0;
2629
2630 cpp = plane_state->hw.fb->format->cpp[0];
2631
2632 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2633 cpp);
2634}
2635
2636static unsigned int
2637ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2638{
2639 if (DISPLAY_VER(dev_priv) >= 8)
2640 return 3072;
2641 else if (DISPLAY_VER(dev_priv) >= 7)
2642 return 768;
2643 else
2644 return 512;
2645}
2646
2647static unsigned int
2648ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2649 int level, bool is_sprite)
2650{
2651 if (DISPLAY_VER(dev_priv) >= 8)
2652 /* BDW primary/sprite plane watermarks */
2653 return level == 0 ? 255 : 2047;
2654 else if (DISPLAY_VER(dev_priv) >= 7)
2655 /* IVB/HSW primary/sprite plane watermarks */
2656 return level == 0 ? 127 : 1023;
2657 else if (!is_sprite)
2658 /* ILK/SNB primary plane watermarks */
2659 return level == 0 ? 127 : 511;
2660 else
2661 /* ILK/SNB sprite plane watermarks */
2662 return level == 0 ? 63 : 255;
2663}
2664
2665static unsigned int
2666ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2667{
2668 if (DISPLAY_VER(dev_priv) >= 7)
2669 return level == 0 ? 63 : 255;
2670 else
2671 return level == 0 ? 31 : 63;
2672}
2673
2674static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2675{
2676 if (DISPLAY_VER(dev_priv) >= 8)
2677 return 31;
2678 else
2679 return 15;
2680}
2681
2682/* Calculate the maximum primary/sprite plane watermark */
2683static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2684 int level,
2685 const struct intel_wm_config *config,
2686 enum intel_ddb_partitioning ddb_partitioning,
2687 bool is_sprite)
2688{
2689 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2690
2691 /* if sprites aren't enabled, sprites get nothing */
2692 if (is_sprite && !config->sprites_enabled)
2693 return 0;
2694
2695 /* HSW allows LP1+ watermarks even with multiple pipes */
2696 if (level == 0 || config->num_pipes_active > 1) {
2697 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2698
2699 /*
2700 * For some reason the non self refresh
2701 * FIFO size is only half of the self
2702 * refresh FIFO size on ILK/SNB.
2703 */
2704 if (DISPLAY_VER(dev_priv) <= 6)
2705 fifo_size /= 2;
2706 }
2707
2708 if (config->sprites_enabled) {
2709 /* level 0 is always calculated with 1:1 split */
2710 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2711 if (is_sprite)
2712 fifo_size *= 5;
2713 fifo_size /= 6;
2714 } else {
2715 fifo_size /= 2;
2716 }
2717 }
2718
2719 /* clamp to max that the registers can hold */
2720 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2721}
2722
2723/* Calculate the maximum cursor plane watermark */
2724static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2725 int level,
2726 const struct intel_wm_config *config)
2727{
2728 /* HSW LP1+ watermarks w/ multiple pipes */
2729 if (level > 0 && config->num_pipes_active > 1)
2730 return 64;
2731
2732 /* otherwise just report max that registers can hold */
2733 return ilk_cursor_wm_reg_max(dev_priv, level);
2734}
2735
2736static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2737 int level,
2738 const struct intel_wm_config *config,
2739 enum intel_ddb_partitioning ddb_partitioning,
2740 struct ilk_wm_maximums *max)
2741{
2742 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2743 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2744 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2745 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2746}
2747
2748static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2749 int level,
2750 struct ilk_wm_maximums *max)
2751{
2752 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2753 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2754 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2755 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2756}
2757
2758static bool ilk_validate_wm_level(int level,
2759 const struct ilk_wm_maximums *max,
2760 struct intel_wm_level *result)
2761{
2762 bool ret;
2763
2764 /* already determined to be invalid? */
2765 if (!result->enable)
2766 return false;
2767
2768 result->enable = result->pri_val <= max->pri &&
2769 result->spr_val <= max->spr &&
2770 result->cur_val <= max->cur;
2771
2772 ret = result->enable;
2773
2774 /*
2775 * HACK until we can pre-compute everything,
2776 * and thus fail gracefully if LP0 watermarks
2777 * are exceeded...
2778 */
2779 if (level == 0 && !result->enable) {
2780 if (result->pri_val > max->pri)
2781 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2782 level, result->pri_val, max->pri);
2783 if (result->spr_val > max->spr)
2784 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2785 level, result->spr_val, max->spr);
2786 if (result->cur_val > max->cur)
2787 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2788 level, result->cur_val, max->cur);
2789
2790 result->pri_val = min_t(u32, result->pri_val, max->pri);
2791 result->spr_val = min_t(u32, result->spr_val, max->spr);
2792 result->cur_val = min_t(u32, result->cur_val, max->cur);
2793 result->enable = true;
2794 }
2795
2796 return ret;
2797}
2798
2799static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2800 const struct intel_crtc *crtc,
2801 int level,
2802 struct intel_crtc_state *crtc_state,
2803 const struct intel_plane_state *pristate,
2804 const struct intel_plane_state *sprstate,
2805 const struct intel_plane_state *curstate,
2806 struct intel_wm_level *result)
2807{
2808 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2809 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2810 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2811
2812 /* WM1+ latency values stored in 0.5us units */
2813 if (level > 0) {
2814 pri_latency *= 5;
2815 spr_latency *= 5;
2816 cur_latency *= 5;
2817 }
2818
2819 if (pristate) {
2820 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2821 pri_latency, level);
2822 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2823 }
2824
2825 if (sprstate)
2826 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2827
2828 if (curstate)
2829 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2830
2831 result->enable = true;
2832}
2833
2834static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2835{
2836 u64 sskpd;
2837
2838 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2839
2840 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2841 if (wm[0] == 0)
2842 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2843 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2844 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2845 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2846 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2847}
2848
2849static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2850{
2851 u32 sskpd;
2852
2853 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2854
2855 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2856 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2857 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2858 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2859}
2860
2861static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2862{
2863 u32 mltr;
2864
2865 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2866
2867 /* ILK primary LP0 latency is 700 ns */
2868 wm[0] = 7;
2869 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2870 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2871}
2872
2873static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2874 u16 wm[5])
2875{
2876 /* ILK sprite LP0 latency is 1300 ns */
2877 if (DISPLAY_VER(dev_priv) == 5)
2878 wm[0] = 13;
2879}
2880
2881static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2882 u16 wm[5])
2883{
2884 /* ILK cursor LP0 latency is 1300 ns */
2885 if (DISPLAY_VER(dev_priv) == 5)
2886 wm[0] = 13;
2887}
2888
2889int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2890{
2891 /* how many WM levels are we expecting */
2892 if (HAS_HW_SAGV_WM(dev_priv))
2893 return 5;
2894 else if (DISPLAY_VER(dev_priv) >= 9)
2895 return 7;
2896 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2897 return 4;
2898 else if (DISPLAY_VER(dev_priv) >= 6)
2899 return 3;
2900 else
2901 return 2;
2902}
2903
2904void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2905 const char *name, const u16 wm[])
2906{
2907 int level, max_level = ilk_wm_max_level(dev_priv);
2908
2909 for (level = 0; level <= max_level; level++) {
2910 unsigned int latency = wm[level];
2911
2912 if (latency == 0) {
2913 drm_dbg_kms(&dev_priv->drm,
2914 "%s WM%d latency not provided\n",
2915 name, level);
2916 continue;
2917 }
2918
2919 /*
2920 * - latencies are in us on gen9.
2921 * - before then, WM1+ latency values are in 0.5us units
2922 */
2923 if (DISPLAY_VER(dev_priv) >= 9)
2924 latency *= 10;
2925 else if (level > 0)
2926 latency *= 5;
2927
2928 drm_dbg_kms(&dev_priv->drm,
2929 "%s WM%d latency %u (%u.%u usec)\n", name, level,
2930 wm[level], latency / 10, latency % 10);
2931 }
2932}
2933
2934static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2935 u16 wm[5], u16 min)
2936{
2937 int level, max_level = ilk_wm_max_level(dev_priv);
2938
2939 if (wm[0] >= min)
2940 return false;
2941
2942 wm[0] = max(wm[0], min);
2943 for (level = 1; level <= max_level; level++)
2944 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2945
2946 return true;
2947}
2948
2949static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2950{
2951 bool changed;
2952
2953 /*
2954 * The BIOS provided WM memory latency values are often
2955 * inadequate for high resolution displays. Adjust them.
2956 */
2957 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2958 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2959 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2960
2961 if (!changed)
2962 return;
2963
2964 drm_dbg_kms(&dev_priv->drm,
2965 "WM latency values increased to avoid potential underruns\n");
2966 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2967 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2968 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2969}
2970
2971static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2972{
2973 /*
2974 * On some SNB machines (Thinkpad X220 Tablet at least)
2975 * LP3 usage can cause vblank interrupts to be lost.
2976 * The DEIIR bit will go high but it looks like the CPU
2977 * never gets interrupted.
2978 *
2979 * It's not clear whether other interrupt source could
2980 * be affected or if this is somehow limited to vblank
2981 * interrupts only. To play it safe we disable LP3
2982 * watermarks entirely.
2983 */
2984 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2985 dev_priv->display.wm.spr_latency[3] == 0 &&
2986 dev_priv->display.wm.cur_latency[3] == 0)
2987 return;
2988
2989 dev_priv->display.wm.pri_latency[3] = 0;
2990 dev_priv->display.wm.spr_latency[3] = 0;
2991 dev_priv->display.wm.cur_latency[3] = 0;
2992
2993 drm_dbg_kms(&dev_priv->drm,
2994 "LP3 watermarks disabled due to potential for lost interrupts\n");
2995 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2996 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2997 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2998}
2999
3000static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3001{
3002 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3003 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3005 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3006 else
3007 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3008
3009 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3010 sizeof(dev_priv->display.wm.pri_latency));
3011 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3012 sizeof(dev_priv->display.wm.pri_latency));
3013
3014 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3015 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3016
3017 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3018 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3019 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3020
3021 if (DISPLAY_VER(dev_priv) == 6) {
3022 snb_wm_latency_quirk(dev_priv);
3023 snb_wm_lp3_irq_quirk(dev_priv);
3024 }
3025}
3026
3027static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3028 struct intel_pipe_wm *pipe_wm)
3029{
3030 /* LP0 watermark maximums depend on this pipe alone */
3031 const struct intel_wm_config config = {
3032 .num_pipes_active = 1,
3033 .sprites_enabled = pipe_wm->sprites_enabled,
3034 .sprites_scaled = pipe_wm->sprites_scaled,
3035 };
3036 struct ilk_wm_maximums max;
3037
3038 /* LP0 watermarks always use 1/2 DDB partitioning */
3039 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3040
3041 /* At least LP0 must be valid */
3042 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3043 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3044 return false;
3045 }
3046
3047 return true;
3048}
3049
3050/* Compute new watermarks for the pipe */
3051static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3052 struct intel_crtc *crtc)
3053{
3054 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3055 struct intel_crtc_state *crtc_state =
3056 intel_atomic_get_new_crtc_state(state, crtc);
3057 struct intel_pipe_wm *pipe_wm;
3058 struct intel_plane *plane;
3059 const struct intel_plane_state *plane_state;
3060 const struct intel_plane_state *pristate = NULL;
3061 const struct intel_plane_state *sprstate = NULL;
3062 const struct intel_plane_state *curstate = NULL;
3063 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3064 struct ilk_wm_maximums max;
3065
3066 pipe_wm = &crtc_state->wm.ilk.optimal;
3067
3068 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3069 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3070 pristate = plane_state;
3071 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3072 sprstate = plane_state;
3073 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3074 curstate = plane_state;
3075 }
3076
3077 pipe_wm->pipe_enabled = crtc_state->hw.active;
3078 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3079 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3080
3081 usable_level = max_level;
3082
3083 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3084 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3085 usable_level = 1;
3086
3087 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3088 if (pipe_wm->sprites_scaled)
3089 usable_level = 0;
3090
3091 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3092 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3093 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3094
3095 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3096 return -EINVAL;
3097
3098 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3099
3100 for (level = 1; level <= usable_level; level++) {
3101 struct intel_wm_level *wm = &pipe_wm->wm[level];
3102
3103 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3104 pristate, sprstate, curstate, wm);
3105
3106 /*
3107 * Disable any watermark level that exceeds the
3108 * register maximums since such watermarks are
3109 * always invalid.
3110 */
3111 if (!ilk_validate_wm_level(level, &max, wm)) {
3112 memset(wm, 0, sizeof(*wm));
3113 break;
3114 }
3115 }
3116
3117 return 0;
3118}
3119
3120/*
3121 * Build a set of 'intermediate' watermark values that satisfy both the old
3122 * state and the new state. These can be programmed to the hardware
3123 * immediately.
3124 */
3125static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3126 struct intel_crtc *crtc)
3127{
3128 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3129 struct intel_crtc_state *new_crtc_state =
3130 intel_atomic_get_new_crtc_state(state, crtc);
3131 const struct intel_crtc_state *old_crtc_state =
3132 intel_atomic_get_old_crtc_state(state, crtc);
3133 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3134 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3135 int level, max_level = ilk_wm_max_level(dev_priv);
3136
3137 /*
3138 * Start with the final, target watermarks, then combine with the
3139 * currently active watermarks to get values that are safe both before
3140 * and after the vblank.
3141 */
3142 *a = new_crtc_state->wm.ilk.optimal;
3143 if (!new_crtc_state->hw.active ||
3144 intel_crtc_needs_modeset(new_crtc_state) ||
3145 state->skip_intermediate_wm)
3146 return 0;
3147
3148 a->pipe_enabled |= b->pipe_enabled;
3149 a->sprites_enabled |= b->sprites_enabled;
3150 a->sprites_scaled |= b->sprites_scaled;
3151
3152 for (level = 0; level <= max_level; level++) {
3153 struct intel_wm_level *a_wm = &a->wm[level];
3154 const struct intel_wm_level *b_wm = &b->wm[level];
3155
3156 a_wm->enable &= b_wm->enable;
3157 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3158 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3159 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3160 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3161 }
3162
3163 /*
3164 * We need to make sure that these merged watermark values are
3165 * actually a valid configuration themselves. If they're not,
3166 * there's no safe way to transition from the old state to
3167 * the new state, so we need to fail the atomic transaction.
3168 */
3169 if (!ilk_validate_pipe_wm(dev_priv, a))
3170 return -EINVAL;
3171
3172 /*
3173 * If our intermediate WM are identical to the final WM, then we can
3174 * omit the post-vblank programming; only update if it's different.
3175 */
3176 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3177 new_crtc_state->wm.need_postvbl_update = true;
3178
3179 return 0;
3180}
3181
3182/*
3183 * Merge the watermarks from all active pipes for a specific level.
3184 */
3185static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3186 int level,
3187 struct intel_wm_level *ret_wm)
3188{
3189 const struct intel_crtc *crtc;
3190
3191 ret_wm->enable = true;
3192
3193 for_each_intel_crtc(&dev_priv->drm, crtc) {
3194 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3195 const struct intel_wm_level *wm = &active->wm[level];
3196
3197 if (!active->pipe_enabled)
3198 continue;
3199
3200 /*
3201 * The watermark values may have been used in the past,
3202 * so we must maintain them in the registers for some
3203 * time even if the level is now disabled.
3204 */
3205 if (!wm->enable)
3206 ret_wm->enable = false;
3207
3208 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3209 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3210 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3211 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3212 }
3213}
3214
3215/*
3216 * Merge all low power watermarks for all active pipes.
3217 */
3218static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3219 const struct intel_wm_config *config,
3220 const struct ilk_wm_maximums *max,
3221 struct intel_pipe_wm *merged)
3222{
3223 int level, max_level = ilk_wm_max_level(dev_priv);
3224 int last_enabled_level = max_level;
3225
3226 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3227 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3228 config->num_pipes_active > 1)
3229 last_enabled_level = 0;
3230
3231 /* ILK: FBC WM must be disabled always */
3232 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3233
3234 /* merge each WM1+ level */
3235 for (level = 1; level <= max_level; level++) {
3236 struct intel_wm_level *wm = &merged->wm[level];
3237
3238 ilk_merge_wm_level(dev_priv, level, wm);
3239
3240 if (level > last_enabled_level)
3241 wm->enable = false;
3242 else if (!ilk_validate_wm_level(level, max, wm))
3243 /* make sure all following levels get disabled */
3244 last_enabled_level = level - 1;
3245
3246 /*
3247 * The spec says it is preferred to disable
3248 * FBC WMs instead of disabling a WM level.
3249 */
3250 if (wm->fbc_val > max->fbc) {
3251 if (wm->enable)
3252 merged->fbc_wm_enabled = false;
3253 wm->fbc_val = 0;
3254 }
3255 }
3256
3257 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3258 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3259 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3260 for (level = 2; level <= max_level; level++) {
3261 struct intel_wm_level *wm = &merged->wm[level];
3262
3263 wm->enable = false;
3264 }
3265 }
3266}
3267
3268static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3269{
3270 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3271 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3272}
3273
3274/* The value we need to program into the WM_LPx latency field */
3275static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3276 int level)
3277{
3278 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3279 return 2 * level;
3280 else
3281 return dev_priv->display.wm.pri_latency[level];
3282}
3283
3284static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3285 const struct intel_pipe_wm *merged,
3286 enum intel_ddb_partitioning partitioning,
3287 struct ilk_wm_values *results)
3288{
3289 struct intel_crtc *crtc;
3290 int level, wm_lp;
3291
3292 results->enable_fbc_wm = merged->fbc_wm_enabled;
3293 results->partitioning = partitioning;
3294
3295 /* LP1+ register values */
3296 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3297 const struct intel_wm_level *r;
3298
3299 level = ilk_wm_lp_to_level(wm_lp, merged);
3300
3301 r = &merged->wm[level];
3302
3303 /*
3304 * Maintain the watermark values even if the level is
3305 * disabled. Doing otherwise could cause underruns.
3306 */
3307 results->wm_lp[wm_lp - 1] =
3308 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3309 WM_LP_PRIMARY(r->pri_val) |
3310 WM_LP_CURSOR(r->cur_val);
3311
3312 if (r->enable)
3313 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3314
3315 if (DISPLAY_VER(dev_priv) >= 8)
3316 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3317 else
3318 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3319
3320 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3321
3322 /*
3323 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3324 * level is disabled. Doing otherwise could cause underruns.
3325 */
3326 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3327 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3328 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3329 }
3330 }
3331
3332 /* LP0 register values */
3333 for_each_intel_crtc(&dev_priv->drm, crtc) {
3334 enum pipe pipe = crtc->pipe;
3335 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3336 const struct intel_wm_level *r = &pipe_wm->wm[0];
3337
3338 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3339 continue;
3340
3341 results->wm_pipe[pipe] =
3342 WM0_PIPE_PRIMARY(r->pri_val) |
3343 WM0_PIPE_SPRITE(r->spr_val) |
3344 WM0_PIPE_CURSOR(r->cur_val);
3345 }
3346}
3347
3348/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3349 * case both are at the same level. Prefer r1 in case they're the same. */
3350static struct intel_pipe_wm *
3351ilk_find_best_result(struct drm_i915_private *dev_priv,
3352 struct intel_pipe_wm *r1,
3353 struct intel_pipe_wm *r2)
3354{
3355 int level, max_level = ilk_wm_max_level(dev_priv);
3356 int level1 = 0, level2 = 0;
3357
3358 for (level = 1; level <= max_level; level++) {
3359 if (r1->wm[level].enable)
3360 level1 = level;
3361 if (r2->wm[level].enable)
3362 level2 = level;
3363 }
3364
3365 if (level1 == level2) {
3366 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3367 return r2;
3368 else
3369 return r1;
3370 } else if (level1 > level2) {
3371 return r1;
3372 } else {
3373 return r2;
3374 }
3375}
3376
3377/* dirty bits used to track which watermarks need changes */
3378#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3379#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3380#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3381#define WM_DIRTY_FBC (1 << 24)
3382#define WM_DIRTY_DDB (1 << 25)
3383
3384static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3385 const struct ilk_wm_values *old,
3386 const struct ilk_wm_values *new)
3387{
3388 unsigned int dirty = 0;
3389 enum pipe pipe;
3390 int wm_lp;
3391
3392 for_each_pipe(dev_priv, pipe) {
3393 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3394 dirty |= WM_DIRTY_PIPE(pipe);
3395 /* Must disable LP1+ watermarks too */
3396 dirty |= WM_DIRTY_LP_ALL;
3397 }
3398 }
3399
3400 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3401 dirty |= WM_DIRTY_FBC;
3402 /* Must disable LP1+ watermarks too */
3403 dirty |= WM_DIRTY_LP_ALL;
3404 }
3405
3406 if (old->partitioning != new->partitioning) {
3407 dirty |= WM_DIRTY_DDB;
3408 /* Must disable LP1+ watermarks too */
3409 dirty |= WM_DIRTY_LP_ALL;
3410 }
3411
3412 /* LP1+ watermarks already deemed dirty, no need to continue */
3413 if (dirty & WM_DIRTY_LP_ALL)
3414 return dirty;
3415
3416 /* Find the lowest numbered LP1+ watermark in need of an update... */
3417 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3418 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3419 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3420 break;
3421 }
3422
3423 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3424 for (; wm_lp <= 3; wm_lp++)
3425 dirty |= WM_DIRTY_LP(wm_lp);
3426
3427 return dirty;
3428}
3429
3430static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3431 unsigned int dirty)
3432{
3433 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3434 bool changed = false;
3435
3436 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3437 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3438 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3439 changed = true;
3440 }
3441 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3442 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3443 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3444 changed = true;
3445 }
3446 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3447 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3448 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3449 changed = true;
3450 }
3451
3452 /*
3453 * Don't touch WM_LP_SPRITE_ENABLE here.
3454 * Doing so could cause underruns.
3455 */
3456
3457 return changed;
3458}
3459
3460/*
3461 * The spec says we shouldn't write when we don't need, because every write
3462 * causes WMs to be re-evaluated, expending some power.
3463 */
3464static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3465 struct ilk_wm_values *results)
3466{
3467 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3468 unsigned int dirty;
3469
3470 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3471 if (!dirty)
3472 return;
3473
3474 _ilk_disable_lp_wm(dev_priv, dirty);
3475
3476 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3477 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3478 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3479 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3480 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3481 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3482
3483 if (dirty & WM_DIRTY_DDB) {
3484 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3485 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3486 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3487 WM_MISC_DATA_PARTITION_5_6);
3488 else
3489 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3490 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3491 DISP_DATA_PARTITION_5_6);
3492 }
3493
3494 if (dirty & WM_DIRTY_FBC)
3495 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3496 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3497
3498 if (dirty & WM_DIRTY_LP(1) &&
3499 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3500 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3501
3502 if (DISPLAY_VER(dev_priv) >= 7) {
3503 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3504 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3505 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3506 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3507 }
3508
3509 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3510 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3511 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3512 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3513 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3514 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3515
3516 dev_priv->display.wm.hw = *results;
3517}
3518
3519bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3520{
3521 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3522}
3523
3524static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3525 struct intel_wm_config *config)
3526{
3527 struct intel_crtc *crtc;
3528
3529 /* Compute the currently _active_ config */
3530 for_each_intel_crtc(&dev_priv->drm, crtc) {
3531 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3532
3533 if (!wm->pipe_enabled)
3534 continue;
3535
3536 config->sprites_enabled |= wm->sprites_enabled;
3537 config->sprites_scaled |= wm->sprites_scaled;
3538 config->num_pipes_active++;
3539 }
3540}
3541
3542static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3543{
3544 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3545 struct ilk_wm_maximums max;
3546 struct intel_wm_config config = {};
3547 struct ilk_wm_values results = {};
3548 enum intel_ddb_partitioning partitioning;
3549
3550 ilk_compute_wm_config(dev_priv, &config);
3551
3552 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3553 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3554
3555 /* 5/6 split only in single pipe config on IVB+ */
3556 if (DISPLAY_VER(dev_priv) >= 7 &&
3557 config.num_pipes_active == 1 && config.sprites_enabled) {
3558 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3559 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3560
3561 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3562 } else {
3563 best_lp_wm = &lp_wm_1_2;
3564 }
3565
3566 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3567 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3568
3569 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3570
3571 ilk_write_wm_values(dev_priv, &results);
3572}
3573
3574static void ilk_initial_watermarks(struct intel_atomic_state *state,
3575 struct intel_crtc *crtc)
3576{
3577 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3578 const struct intel_crtc_state *crtc_state =
3579 intel_atomic_get_new_crtc_state(state, crtc);
3580
3581 mutex_lock(&dev_priv->display.wm.wm_mutex);
3582 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3583 ilk_program_watermarks(dev_priv);
3584 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3585}
3586
3587static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3588 struct intel_crtc *crtc)
3589{
3590 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3591 const struct intel_crtc_state *crtc_state =
3592 intel_atomic_get_new_crtc_state(state, crtc);
3593
3594 if (!crtc_state->wm.need_postvbl_update)
3595 return;
3596
3597 mutex_lock(&dev_priv->display.wm.wm_mutex);
3598 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3599 ilk_program_watermarks(dev_priv);
3600 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3601}
3602
3603static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3604{
3605 struct drm_device *dev = crtc->base.dev;
3606 struct drm_i915_private *dev_priv = to_i915(dev);
3607 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3608 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3609 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3610 enum pipe pipe = crtc->pipe;
3611
3612 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3613
3614 memset(active, 0, sizeof(*active));
3615
3616 active->pipe_enabled = crtc->active;
3617
3618 if (active->pipe_enabled) {
3619 u32 tmp = hw->wm_pipe[pipe];
3620
3621 /*
3622 * For active pipes LP0 watermark is marked as
3623 * enabled, and LP1+ watermaks as disabled since
3624 * we can't really reverse compute them in case
3625 * multiple pipes are active.
3626 */
3627 active->wm[0].enable = true;
3628 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3629 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3630 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3631 } else {
3632 int level, max_level = ilk_wm_max_level(dev_priv);
3633
3634 /*
3635 * For inactive pipes, all watermark levels
3636 * should be marked as enabled but zeroed,
3637 * which is what we'd compute them to.
3638 */
3639 for (level = 0; level <= max_level; level++)
3640 active->wm[level].enable = true;
3641 }
3642
3643 crtc->wm.active.ilk = *active;
3644}
3645
3646#define _FW_WM(value, plane) \
3647 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3648#define _FW_WM_VLV(value, plane) \
3649 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3650
3651static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3652 struct g4x_wm_values *wm)
3653{
3654 u32 tmp;
3655
3656 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3657 wm->sr.plane = _FW_WM(tmp, SR);
3658 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3659 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3660 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3661
3662 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3663 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3664 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3665 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3666 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3667 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3668 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3669
3670 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3671 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3672 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3673 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3674 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3675}
3676
3677static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3678 struct vlv_wm_values *wm)
3679{
3680 enum pipe pipe;
3681 u32 tmp;
3682
3683 for_each_pipe(dev_priv, pipe) {
3684 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3685
3686 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3687 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3688 wm->ddl[pipe].plane[PLANE_CURSOR] =
3689 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3690 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3691 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3692 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3693 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3694 }
3695
3696 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3697 wm->sr.plane = _FW_WM(tmp, SR);
3698 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3699 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3700 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3701
3702 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3703 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3704 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3705 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3706
3707 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3708 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3709
3710 if (IS_CHERRYVIEW(dev_priv)) {
3711 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3712 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3713 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3714
3715 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3716 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3717 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3718
3719 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3720 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3721 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3722
3723 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3724 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3725 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3726 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3727 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3728 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3729 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3730 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3731 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3732 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3733 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3734 } else {
3735 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3736 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3737 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3738
3739 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3740 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3741 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3742 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3743 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3744 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3745 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3746 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3747 }
3748}
3749
3750#undef _FW_WM
3751#undef _FW_WM_VLV
3752
3753void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3754{
3755 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3756 struct intel_crtc *crtc;
3757
3758 g4x_read_wm_values(dev_priv, wm);
3759
3760 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3761
3762 for_each_intel_crtc(&dev_priv->drm, crtc) {
3763 struct intel_crtc_state *crtc_state =
3764 to_intel_crtc_state(crtc->base.state);
3765 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3766 struct g4x_pipe_wm *raw;
3767 enum pipe pipe = crtc->pipe;
3768 enum plane_id plane_id;
3769 int level, max_level;
3770
3771 active->cxsr = wm->cxsr;
3772 active->hpll_en = wm->hpll_en;
3773 active->fbc_en = wm->fbc_en;
3774
3775 active->sr = wm->sr;
3776 active->hpll = wm->hpll;
3777
3778 for_each_plane_id_on_crtc(crtc, plane_id) {
3779 active->wm.plane[plane_id] =
3780 wm->pipe[pipe].plane[plane_id];
3781 }
3782
3783 if (wm->cxsr && wm->hpll_en)
3784 max_level = G4X_WM_LEVEL_HPLL;
3785 else if (wm->cxsr)
3786 max_level = G4X_WM_LEVEL_SR;
3787 else
3788 max_level = G4X_WM_LEVEL_NORMAL;
3789
3790 level = G4X_WM_LEVEL_NORMAL;
3791 raw = &crtc_state->wm.g4x.raw[level];
3792 for_each_plane_id_on_crtc(crtc, plane_id)
3793 raw->plane[plane_id] = active->wm.plane[plane_id];
3794
3795 level = G4X_WM_LEVEL_SR;
3796 if (level > max_level)
3797 goto out;
3798
3799 raw = &crtc_state->wm.g4x.raw[level];
3800 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3801 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3802 raw->plane[PLANE_SPRITE0] = 0;
3803 raw->fbc = active->sr.fbc;
3804
3805 level = G4X_WM_LEVEL_HPLL;
3806 if (level > max_level)
3807 goto out;
3808
3809 raw = &crtc_state->wm.g4x.raw[level];
3810 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3811 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3812 raw->plane[PLANE_SPRITE0] = 0;
3813 raw->fbc = active->hpll.fbc;
3814
3815 level++;
3816 out:
3817 for_each_plane_id_on_crtc(crtc, plane_id)
3818 g4x_raw_plane_wm_set(crtc_state, level,
3819 plane_id, USHRT_MAX);
3820 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3821
3822 g4x_invalidate_wms(crtc, active, level);
3823
3824 crtc_state->wm.g4x.optimal = *active;
3825 crtc_state->wm.g4x.intermediate = *active;
3826
3827 drm_dbg_kms(&dev_priv->drm,
3828 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3829 pipe_name(pipe),
3830 wm->pipe[pipe].plane[PLANE_PRIMARY],
3831 wm->pipe[pipe].plane[PLANE_CURSOR],
3832 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3833 }
3834
3835 drm_dbg_kms(&dev_priv->drm,
3836 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3837 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3838 drm_dbg_kms(&dev_priv->drm,
3839 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3840 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3841 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3842 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3843 str_yes_no(wm->fbc_en));
3844}
3845
3846void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3847{
3848 struct intel_plane *plane;
3849 struct intel_crtc *crtc;
3850
3851 mutex_lock(&dev_priv->display.wm.wm_mutex);
3852
3853 for_each_intel_plane(&dev_priv->drm, plane) {
3854 struct intel_crtc *crtc =
3855 intel_crtc_for_pipe(dev_priv, plane->pipe);
3856 struct intel_crtc_state *crtc_state =
3857 to_intel_crtc_state(crtc->base.state);
3858 struct intel_plane_state *plane_state =
3859 to_intel_plane_state(plane->base.state);
3860 enum plane_id plane_id = plane->id;
3861 int level, num_levels = intel_wm_num_levels(dev_priv);
3862
3863 if (plane_state->uapi.visible)
3864 continue;
3865
3866 for (level = 0; level < num_levels; level++) {
3867 struct g4x_pipe_wm *raw =
3868 &crtc_state->wm.g4x.raw[level];
3869
3870 raw->plane[plane_id] = 0;
3871
3872 if (plane_id == PLANE_PRIMARY)
3873 raw->fbc = 0;
3874 }
3875 }
3876
3877 for_each_intel_crtc(&dev_priv->drm, crtc) {
3878 struct intel_crtc_state *crtc_state =
3879 to_intel_crtc_state(crtc->base.state);
3880 int ret;
3881
3882 ret = _g4x_compute_pipe_wm(crtc_state);
3883 drm_WARN_ON(&dev_priv->drm, ret);
3884
3885 crtc_state->wm.g4x.intermediate =
3886 crtc_state->wm.g4x.optimal;
3887 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3888 }
3889
3890 g4x_program_watermarks(dev_priv);
3891
3892 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3893}
3894
3895void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3896{
3897 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3898 struct intel_crtc *crtc;
3899 u32 val;
3900
3901 vlv_read_wm_values(dev_priv, wm);
3902
3903 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3904 wm->level = VLV_WM_LEVEL_PM2;
3905
3906 if (IS_CHERRYVIEW(dev_priv)) {
3907 vlv_punit_get(dev_priv);
3908
3909 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3910 if (val & DSP_MAXFIFO_PM5_ENABLE)
3911 wm->level = VLV_WM_LEVEL_PM5;
3912
3913 /*
3914 * If DDR DVFS is disabled in the BIOS, Punit
3915 * will never ack the request. So if that happens
3916 * assume we don't have to enable/disable DDR DVFS
3917 * dynamically. To test that just set the REQ_ACK
3918 * bit to poke the Punit, but don't change the
3919 * HIGH/LOW bits so that we don't actually change
3920 * the current state.
3921 */
3922 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3923 val |= FORCE_DDR_FREQ_REQ_ACK;
3924 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3925
3926 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3927 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3928 drm_dbg_kms(&dev_priv->drm,
3929 "Punit not acking DDR DVFS request, "
3930 "assuming DDR DVFS is disabled\n");
3931 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3932 } else {
3933 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3934 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3935 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3936 }
3937
3938 vlv_punit_put(dev_priv);
3939 }
3940
3941 for_each_intel_crtc(&dev_priv->drm, crtc) {
3942 struct intel_crtc_state *crtc_state =
3943 to_intel_crtc_state(crtc->base.state);
3944 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3945 const struct vlv_fifo_state *fifo_state =
3946 &crtc_state->wm.vlv.fifo_state;
3947 enum pipe pipe = crtc->pipe;
3948 enum plane_id plane_id;
3949 int level;
3950
3951 vlv_get_fifo_size(crtc_state);
3952
3953 active->num_levels = wm->level + 1;
3954 active->cxsr = wm->cxsr;
3955
3956 for (level = 0; level < active->num_levels; level++) {
3957 struct g4x_pipe_wm *raw =
3958 &crtc_state->wm.vlv.raw[level];
3959
3960 active->sr[level].plane = wm->sr.plane;
3961 active->sr[level].cursor = wm->sr.cursor;
3962
3963 for_each_plane_id_on_crtc(crtc, plane_id) {
3964 active->wm[level].plane[plane_id] =
3965 wm->pipe[pipe].plane[plane_id];
3966
3967 raw->plane[plane_id] =
3968 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3969 fifo_state->plane[plane_id]);
3970 }
3971 }
3972
3973 for_each_plane_id_on_crtc(crtc, plane_id)
3974 vlv_raw_plane_wm_set(crtc_state, level,
3975 plane_id, USHRT_MAX);
3976 vlv_invalidate_wms(crtc, active, level);
3977
3978 crtc_state->wm.vlv.optimal = *active;
3979 crtc_state->wm.vlv.intermediate = *active;
3980
3981 drm_dbg_kms(&dev_priv->drm,
3982 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3983 pipe_name(pipe),
3984 wm->pipe[pipe].plane[PLANE_PRIMARY],
3985 wm->pipe[pipe].plane[PLANE_CURSOR],
3986 wm->pipe[pipe].plane[PLANE_SPRITE0],
3987 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3988 }
3989
3990 drm_dbg_kms(&dev_priv->drm,
3991 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3992 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3993}
3994
3995void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3996{
3997 struct intel_plane *plane;
3998 struct intel_crtc *crtc;
3999
4000 mutex_lock(&dev_priv->display.wm.wm_mutex);
4001
4002 for_each_intel_plane(&dev_priv->drm, plane) {
4003 struct intel_crtc *crtc =
4004 intel_crtc_for_pipe(dev_priv, plane->pipe);
4005 struct intel_crtc_state *crtc_state =
4006 to_intel_crtc_state(crtc->base.state);
4007 struct intel_plane_state *plane_state =
4008 to_intel_plane_state(plane->base.state);
4009 enum plane_id plane_id = plane->id;
4010 int level, num_levels = intel_wm_num_levels(dev_priv);
4011
4012 if (plane_state->uapi.visible)
4013 continue;
4014
4015 for (level = 0; level < num_levels; level++) {
4016 struct g4x_pipe_wm *raw =
4017 &crtc_state->wm.vlv.raw[level];
4018
4019 raw->plane[plane_id] = 0;
4020 }
4021 }
4022
4023 for_each_intel_crtc(&dev_priv->drm, crtc) {
4024 struct intel_crtc_state *crtc_state =
4025 to_intel_crtc_state(crtc->base.state);
4026 int ret;
4027
4028 ret = _vlv_compute_pipe_wm(crtc_state);
4029 drm_WARN_ON(&dev_priv->drm, ret);
4030
4031 crtc_state->wm.vlv.intermediate =
4032 crtc_state->wm.vlv.optimal;
4033 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4034 }
4035
4036 vlv_program_watermarks(dev_priv);
4037
4038 mutex_unlock(&dev_priv->display.wm.wm_mutex);
4039}
4040
4041/*
4042 * FIXME should probably kill this and improve
4043 * the real watermark readout/sanitation instead
4044 */
4045static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4046{
4047 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4048 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4049 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4050
4051 /*
4052 * Don't touch WM_LP_SPRITE_ENABLE here.
4053 * Doing so could cause underruns.
4054 */
4055}
4056
4057void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4058{
4059 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4060 struct intel_crtc *crtc;
4061
4062 ilk_init_lp_watermarks(dev_priv);
4063
4064 for_each_intel_crtc(&dev_priv->drm, crtc)
4065 ilk_pipe_wm_get_hw_state(crtc);
4066
4067 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4068 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4069 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4070
4071 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4072 if (DISPLAY_VER(dev_priv) >= 7) {
4073 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4074 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4075 }
4076
4077 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4078 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4079 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4080 else if (IS_IVYBRIDGE(dev_priv))
4081 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4082 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4083
4084 hw->enable_fbc_wm =
4085 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4086}
4087
4088static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4089{
4090 /*
4091 * On Ibex Peak and Cougar Point, we need to disable clock
4092 * gating for the panel power sequencer or it will fail to
4093 * start up when no ports are active.
4094 */
4095 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4096}
4097
4098static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4099{
4100 enum pipe pipe;
4101
4102 for_each_pipe(dev_priv, pipe) {
4103 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4104
4105 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4106 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4107 }
4108}
4109
4110static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4111{
4112 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4113
4114 /*
4115 * Required for FBC
4116 * WaFbcDisableDpfcClockGating:ilk
4117 */
4118 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4119 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4120 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4121
4122 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4123 MARIUNIT_CLOCK_GATE_DISABLE |
4124 SVSMUNIT_CLOCK_GATE_DISABLE);
4125 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4126 VFMUNIT_CLOCK_GATE_DISABLE);
4127
4128 /*
4129 * According to the spec the following bits should be set in
4130 * order to enable memory self-refresh
4131 * The bit 22/21 of 0x42004
4132 * The bit 5 of 0x42020
4133 * The bit 15 of 0x45000
4134 */
4135 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4136 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4137 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4138 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4139 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4140 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4141 DISP_FBC_WM_DIS));
4142
4143 /*
4144 * Based on the document from hardware guys the following bits
4145 * should be set unconditionally in order to enable FBC.
4146 * The bit 22 of 0x42000
4147 * The bit 22 of 0x42004
4148 * The bit 7,8,9 of 0x42020.
4149 */
4150 if (IS_IRONLAKE_M(dev_priv)) {
4151 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4152 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4153 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4154 }
4155
4156 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4157
4158 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4159
4160 g4x_disable_trickle_feed(dev_priv);
4161
4162 ibx_init_clock_gating(dev_priv);
4163}
4164
4165static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4166{
4167 enum pipe pipe;
4168 u32 val;
4169
4170 /*
4171 * On Ibex Peak and Cougar Point, we need to disable clock
4172 * gating for the panel power sequencer or it will fail to
4173 * start up when no ports are active.
4174 */
4175 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4176 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4177 PCH_CPUNIT_CLOCK_GATE_DISABLE);
4178 intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4179 /* The below fixes the weird display corruption, a few pixels shifted
4180 * downward, on (only) LVDS of some HP laptops with IVY.
4181 */
4182 for_each_pipe(dev_priv, pipe) {
4183 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4184 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4185 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4186 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4187 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4188 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4189 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4190 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4191 }
4192 /* WADP0ClockGatingDisable */
4193 for_each_pipe(dev_priv, pipe) {
4194 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4195 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4196 }
4197}
4198
4199static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4200{
4201 u32 tmp;
4202
4203 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4204 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4205 drm_dbg_kms(&dev_priv->drm,
4206 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4207 tmp);
4208}
4209
4210static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4211{
4212 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4213
4214 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4215
4216 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4217
4218 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4219 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4220 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4221 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4222
4223 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4224 * gating disable must be set. Failure to set it results in
4225 * flickering pixels due to Z write ordering failures after
4226 * some amount of runtime in the Mesa "fire" demo, and Unigine
4227 * Sanctuary and Tropics, and apparently anything else with
4228 * alpha test or pixel discard.
4229 *
4230 * According to the spec, bit 11 (RCCUNIT) must also be set,
4231 * but we didn't debug actual testcases to find it out.
4232 *
4233 * WaDisableRCCUnitClockGating:snb
4234 * WaDisableRCPBUnitClockGating:snb
4235 */
4236 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4237 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4238 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4239
4240 /*
4241 * According to the spec the following bits should be
4242 * set in order to enable memory self-refresh and fbc:
4243 * The bit21 and bit22 of 0x42000
4244 * The bit21 and bit22 of 0x42004
4245 * The bit5 and bit7 of 0x42020
4246 * The bit14 of 0x70180
4247 * The bit14 of 0x71180
4248 *
4249 * WaFbcAsynchFlipDisableFbcQueue:snb
4250 */
4251 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4252 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4253 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4254 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4255 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4256 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4257 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4258 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4259 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4260 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4261
4262 g4x_disable_trickle_feed(dev_priv);
4263
4264 cpt_init_clock_gating(dev_priv);
4265
4266 gen6_check_mch_setup(dev_priv);
4267}
4268
4269static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4270{
4271 /*
4272 * TODO: this bit should only be enabled when really needed, then
4273 * disabled when not needed anymore in order to save power.
4274 */
4275 if (HAS_PCH_LPT_LP(dev_priv))
4276 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4277 0, PCH_LP_PARTITION_LEVEL_DISABLE);
4278
4279 /* WADPOClockGatingDisable:hsw */
4280 intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4281 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4282}
4283
4284static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4285{
4286 if (HAS_PCH_LPT_LP(dev_priv)) {
4287 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4288
4289 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4290 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4291 }
4292}
4293
4294static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4295 int general_prio_credits,
4296 int high_prio_credits)
4297{
4298 u32 misccpctl;
4299 u32 val;
4300
4301 /* WaTempDisableDOPClkGating:bdw */
4302 misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4303 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4304
4305 val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4306 val &= ~L3_PRIO_CREDITS_MASK;
4307 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4308 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4309 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4310
4311 /*
4312 * Wait at least 100 clocks before re-enabling clock gating.
4313 * See the definition of L3SQCREG1 in BSpec.
4314 */
4315 intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4316 udelay(1);
4317 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
4318}
4319
4320static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4321{
4322 /* Wa_1409120013:icl,ehl */
4323 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4324 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4325
4326 /*Wa_14010594013:icl, ehl */
4327 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4328 0, ICL_DELAY_PMRSP);
4329}
4330
4331static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4332{
4333 /* Wa_1409120013 */
4334 if (DISPLAY_VER(dev_priv) == 12)
4335 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4336 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4337
4338 /* Wa_1409825376:tgl (pre-prod)*/
4339 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4340 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4341
4342 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4343 if (DISPLAY_VER(dev_priv) == 12)
4344 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4345 CLKREQ_POLICY_MEM_UP_OVRD, 0);
4346}
4347
4348static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4349{
4350 gen12lp_init_clock_gating(dev_priv);
4351
4352 /* Wa_22011091694:adlp */
4353 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4354
4355 /* Bspec/49189 Initialize Sequence */
4356 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4357}
4358
4359static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4360{
4361 gen12lp_init_clock_gating(dev_priv);
4362
4363 /* Wa_1409836686:dg1[a0] */
4364 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4365 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4366}
4367
4368static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4369{
4370 /* Wa_22010146351:xehpsdv */
4371 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4372 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4373}
4374
4375static void dg2_init_clock_gating(struct drm_i915_private *i915)
4376{
4377 /* Wa_22010954014:dg2 */
4378 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4379 SGSI_SIDECLK_DIS);
4380
4381 /*
4382 * Wa_14010733611:dg2_g10
4383 * Wa_22010146351:dg2_g10
4384 */
4385 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4386 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4387 SGR_DIS | SGGI_DIS);
4388}
4389
4390static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4391{
4392 /* Wa_14012385139:pvc */
4393 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4394 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4395
4396 /* Wa_22010954014:pvc */
4397 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4398 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4399}
4400
4401static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4402{
4403 if (!HAS_PCH_CNP(dev_priv))
4404 return;
4405
4406 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4407 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4408}
4409
4410static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4411{
4412 cnp_init_clock_gating(dev_priv);
4413 gen9_init_clock_gating(dev_priv);
4414
4415 /* WAC6entrylatency:cfl */
4416 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4417
4418 /*
4419 * WaFbcTurnOffFbcWatermark:cfl
4420 * Display WA #0562: cfl
4421 */
4422 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4423
4424 /*
4425 * WaFbcNukeOnHostModify:cfl
4426 * Display WA #0873: cfl
4427 */
4428 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4429 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4430}
4431
4432static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4433{
4434 gen9_init_clock_gating(dev_priv);
4435
4436 /* WAC6entrylatency:kbl */
4437 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4438
4439 /* WaDisableSDEUnitClockGating:kbl */
4440 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4441 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4442 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4443
4444 /* WaDisableGamClockGating:kbl */
4445 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4446 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4447 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4448
4449 /*
4450 * WaFbcTurnOffFbcWatermark:kbl
4451 * Display WA #0562: kbl
4452 */
4453 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4454
4455 /*
4456 * WaFbcNukeOnHostModify:kbl
4457 * Display WA #0873: kbl
4458 */
4459 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4460 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4461}
4462
4463static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4464{
4465 gen9_init_clock_gating(dev_priv);
4466
4467 /* WaDisableDopClockGating:skl */
4468 intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4469 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4470
4471 /* WAC6entrylatency:skl */
4472 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4473
4474 /*
4475 * WaFbcTurnOffFbcWatermark:skl
4476 * Display WA #0562: skl
4477 */
4478 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4479
4480 /*
4481 * WaFbcNukeOnHostModify:skl
4482 * Display WA #0873: skl
4483 */
4484 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4485 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4486
4487 /*
4488 * WaFbcHighMemBwCorruptionAvoidance:skl
4489 * Display WA #0883: skl
4490 */
4491 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4492}
4493
4494static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4495{
4496 enum pipe pipe;
4497
4498 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4499 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4500
4501 /* WaSwitchSolVfFArbitrationPriority:bdw */
4502 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4503
4504 /* WaPsrDPAMaskVBlankInSRD:bdw */
4505 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4506
4507 for_each_pipe(dev_priv, pipe) {
4508 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4509 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4510 0, BDW_DPRS_MASK_VBLANK_SRD);
4511 }
4512
4513 /* WaVSRefCountFullforceMissDisable:bdw */
4514 /* WaDSRefCountFullforceMissDisable:bdw */
4515 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4516 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4517
4518 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4519 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4520
4521 /* WaDisableSDEUnitClockGating:bdw */
4522 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4523
4524 /* WaProgramL3SqcReg1Default:bdw */
4525 gen8_set_l3sqc_credits(dev_priv, 30, 2);
4526
4527 /* WaKVMNotificationOnConfigChange:bdw */
4528 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4529 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4530
4531 lpt_init_clock_gating(dev_priv);
4532
4533 /* WaDisableDopClockGating:bdw
4534 *
4535 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4536 * clock gating.
4537 */
4538 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4539}
4540
4541static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4542{
4543 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4544 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4545
4546 /* This is required by WaCatErrorRejectionIssue:hsw */
4547 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4548 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4549
4550 /* WaSwitchSolVfFArbitrationPriority:hsw */
4551 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4552
4553 lpt_init_clock_gating(dev_priv);
4554}
4555
4556static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4557{
4558 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4559
4560 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4561 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4562
4563 /* WaDisableBackToBackFlipFix:ivb */
4564 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4565 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4566 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4567
4568 if (IS_IVB_GT1(dev_priv))
4569 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4570 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4571 else {
4572 /* must write both registers */
4573 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4574 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4575 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4576 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4577 }
4578
4579 /*
4580 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4581 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4582 */
4583 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4584 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4585
4586 /* This is required by WaCatErrorRejectionIssue:ivb */
4587 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4588 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4589
4590 g4x_disable_trickle_feed(dev_priv);
4591
4592 intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4593 GEN6_MBC_SNPCR_MED);
4594
4595 if (!HAS_PCH_NOP(dev_priv))
4596 cpt_init_clock_gating(dev_priv);
4597
4598 gen6_check_mch_setup(dev_priv);
4599}
4600
4601static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4602{
4603 /* WaDisableBackToBackFlipFix:vlv */
4604 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4605 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4606 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4607
4608 /* WaDisableDopClockGating:vlv */
4609 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4610 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4611
4612 /* This is required by WaCatErrorRejectionIssue:vlv */
4613 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4614 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4615
4616 /*
4617 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4618 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4619 */
4620 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4621 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4622
4623 /* WaDisableL3Bank2xClockGate:vlv
4624 * Disabling L3 clock gating- MMIO 940c[25] = 1
4625 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4626 intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4627
4628 /*
4629 * WaDisableVLVClockGating_VBIIssue:vlv
4630 * Disable clock gating on th GCFG unit to prevent a delay
4631 * in the reporting of vblank events.
4632 */
4633 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4634}
4635
4636static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4637{
4638 /* WaVSRefCountFullforceMissDisable:chv */
4639 /* WaDSRefCountFullforceMissDisable:chv */
4640 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4641 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4642
4643 /* WaDisableSemaphoreAndSyncFlipWait:chv */
4644 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4645 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4646
4647 /* WaDisableCSUnitClockGating:chv */
4648 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4649
4650 /* WaDisableSDEUnitClockGating:chv */
4651 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4652
4653 /*
4654 * WaProgramL3SqcReg1Default:chv
4655 * See gfxspecs/Related Documents/Performance Guide/
4656 * LSQC Setting Recommendations.
4657 */
4658 gen8_set_l3sqc_credits(dev_priv, 38, 2);
4659}
4660
4661static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4662{
4663 u32 dspclk_gate;
4664
4665 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4666 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4667 GS_UNIT_CLOCK_GATE_DISABLE |
4668 CL_UNIT_CLOCK_GATE_DISABLE);
4669 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4670 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4671 OVRUNIT_CLOCK_GATE_DISABLE |
4672 OVCUNIT_CLOCK_GATE_DISABLE;
4673 if (IS_GM45(dev_priv))
4674 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4675 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4676
4677 g4x_disable_trickle_feed(dev_priv);
4678}
4679
4680static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4681{
4682 struct intel_uncore *uncore = &dev_priv->uncore;
4683
4684 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4685 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4686 intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4687 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4688 intel_uncore_write16(uncore, DEUC, 0);
4689 intel_uncore_write(uncore,
4690 MI_ARB_STATE,
4691 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4692}
4693
4694static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4695{
4696 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4697 I965_RCC_CLOCK_GATE_DISABLE |
4698 I965_RCPB_CLOCK_GATE_DISABLE |
4699 I965_ISC_CLOCK_GATE_DISABLE |
4700 I965_FBC_CLOCK_GATE_DISABLE);
4701 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4702 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4703 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4704}
4705
4706static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4707{
4708 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4709
4710 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4711 DSTATE_DOT_CLOCK_GATING;
4712 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4713
4714 if (IS_PINEVIEW(dev_priv))
4715 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4716 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4717
4718 /* IIR "flip pending" means done if this bit is set */
4719 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4720 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4721
4722 /* interrupts should cause a wake up from C3 */
4723 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4724
4725 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4726 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4727
4728 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4729 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4730}
4731
4732static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4733{
4734 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4735
4736 /* interrupts should cause a wake up from C3 */
4737 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4738 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4739
4740 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4741 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4742
4743 /*
4744 * Have FBC ignore 3D activity since we use software
4745 * render tracking, and otherwise a pure 3D workload
4746 * (even if it just renders a single frame and then does
4747 * abosultely nothing) would not allow FBC to recompress
4748 * until a 2D blit occurs.
4749 */
4750 intel_uncore_write(&dev_priv->uncore, SCPD0,
4751 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4752}
4753
4754static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4755{
4756 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4757 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4758 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4759}
4760
4761void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4762{
4763 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4764}
4765
4766void intel_suspend_hw(struct drm_i915_private *dev_priv)
4767{
4768 if (HAS_PCH_LPT(dev_priv))
4769 lpt_suspend_hw(dev_priv);
4770}
4771
4772static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4773{
4774 drm_dbg_kms(&dev_priv->drm,
4775 "No clock gating settings or workarounds applied.\n");
4776}
4777
4778#define CG_FUNCS(platform) \
4779static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4780 .init_clock_gating = platform##_init_clock_gating, \
4781}
4782
4783CG_FUNCS(pvc);
4784CG_FUNCS(dg2);
4785CG_FUNCS(xehpsdv);
4786CG_FUNCS(adlp);
4787CG_FUNCS(dg1);
4788CG_FUNCS(gen12lp);
4789CG_FUNCS(icl);
4790CG_FUNCS(cfl);
4791CG_FUNCS(skl);
4792CG_FUNCS(kbl);
4793CG_FUNCS(bxt);
4794CG_FUNCS(glk);
4795CG_FUNCS(bdw);
4796CG_FUNCS(chv);
4797CG_FUNCS(hsw);
4798CG_FUNCS(ivb);
4799CG_FUNCS(vlv);
4800CG_FUNCS(gen6);
4801CG_FUNCS(ilk);
4802CG_FUNCS(g4x);
4803CG_FUNCS(i965gm);
4804CG_FUNCS(i965g);
4805CG_FUNCS(gen3);
4806CG_FUNCS(i85x);
4807CG_FUNCS(i830);
4808CG_FUNCS(nop);
4809#undef CG_FUNCS
4810
4811/**
4812 * intel_init_clock_gating_hooks - setup the clock gating hooks
4813 * @dev_priv: device private
4814 *
4815 * Setup the hooks that configure which clocks of a given platform can be
4816 * gated and also apply various GT and display specific workarounds for these
4817 * platforms. Note that some GT specific workarounds are applied separately
4818 * when GPU contexts or batchbuffers start their execution.
4819 */
4820void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4821{
4822 if (IS_PONTEVECCHIO(dev_priv))
4823 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4824 else if (IS_DG2(dev_priv))
4825 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4826 else if (IS_XEHPSDV(dev_priv))
4827 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4828 else if (IS_ALDERLAKE_P(dev_priv))
4829 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4830 else if (IS_DG1(dev_priv))
4831 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4832 else if (GRAPHICS_VER(dev_priv) == 12)
4833 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4834 else if (GRAPHICS_VER(dev_priv) == 11)
4835 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4836 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4837 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4838 else if (IS_SKYLAKE(dev_priv))
4839 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4840 else if (IS_KABYLAKE(dev_priv))
4841 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4842 else if (IS_BROXTON(dev_priv))
4843 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4844 else if (IS_GEMINILAKE(dev_priv))
4845 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4846 else if (IS_BROADWELL(dev_priv))
4847 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4848 else if (IS_CHERRYVIEW(dev_priv))
4849 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4850 else if (IS_HASWELL(dev_priv))
4851 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4852 else if (IS_IVYBRIDGE(dev_priv))
4853 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4854 else if (IS_VALLEYVIEW(dev_priv))
4855 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4856 else if (GRAPHICS_VER(dev_priv) == 6)
4857 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4858 else if (GRAPHICS_VER(dev_priv) == 5)
4859 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4860 else if (IS_G4X(dev_priv))
4861 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4862 else if (IS_I965GM(dev_priv))
4863 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4864 else if (IS_I965G(dev_priv))
4865 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4866 else if (GRAPHICS_VER(dev_priv) == 3)
4867 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4868 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4869 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4870 else if (GRAPHICS_VER(dev_priv) == 2)
4871 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4872 else {
4873 MISSING_CASE(INTEL_DEVID(dev_priv));
4874 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4875 }
4876}
4877
4878static const struct intel_wm_funcs ilk_wm_funcs = {
4879 .compute_pipe_wm = ilk_compute_pipe_wm,
4880 .compute_intermediate_wm = ilk_compute_intermediate_wm,
4881 .initial_watermarks = ilk_initial_watermarks,
4882 .optimize_watermarks = ilk_optimize_watermarks,
4883};
4884
4885static const struct intel_wm_funcs vlv_wm_funcs = {
4886 .compute_pipe_wm = vlv_compute_pipe_wm,
4887 .compute_intermediate_wm = vlv_compute_intermediate_wm,
4888 .initial_watermarks = vlv_initial_watermarks,
4889 .optimize_watermarks = vlv_optimize_watermarks,
4890 .atomic_update_watermarks = vlv_atomic_update_fifo,
4891};
4892
4893static const struct intel_wm_funcs g4x_wm_funcs = {
4894 .compute_pipe_wm = g4x_compute_pipe_wm,
4895 .compute_intermediate_wm = g4x_compute_intermediate_wm,
4896 .initial_watermarks = g4x_initial_watermarks,
4897 .optimize_watermarks = g4x_optimize_watermarks,
4898};
4899
4900static const struct intel_wm_funcs pnv_wm_funcs = {
4901 .update_wm = pnv_update_wm,
4902};
4903
4904static const struct intel_wm_funcs i965_wm_funcs = {
4905 .update_wm = i965_update_wm,
4906};
4907
4908static const struct intel_wm_funcs i9xx_wm_funcs = {
4909 .update_wm = i9xx_update_wm,
4910};
4911
4912static const struct intel_wm_funcs i845_wm_funcs = {
4913 .update_wm = i845_update_wm,
4914};
4915
4916static const struct intel_wm_funcs nop_funcs = {
4917};
4918
4919/* Set up chip specific power management-related functions */
4920void intel_init_pm(struct drm_i915_private *dev_priv)
4921{
4922 if (DISPLAY_VER(dev_priv) >= 9) {
4923 skl_wm_init(dev_priv);
4924 return;
4925 }
4926
4927 /* For cxsr */
4928 if (IS_PINEVIEW(dev_priv))
4929 pnv_get_mem_freq(dev_priv);
4930 else if (GRAPHICS_VER(dev_priv) == 5)
4931 ilk_get_mem_freq(dev_priv);
4932
4933 /* For FIFO watermark updates */
4934 if (HAS_PCH_SPLIT(dev_priv)) {
4935 ilk_setup_wm_latency(dev_priv);
4936
4937 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4938 dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4939 (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4940 dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4941 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4942 } else {
4943 drm_dbg_kms(&dev_priv->drm,
4944 "Failed to read display plane latency. "
4945 "Disable CxSR\n");
4946 dev_priv->display.funcs.wm = &nop_funcs;
4947 }
4948 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4949 vlv_setup_wm_latency(dev_priv);
4950 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4951 } else if (IS_G4X(dev_priv)) {
4952 g4x_setup_wm_latency(dev_priv);
4953 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4954 } else if (IS_PINEVIEW(dev_priv)) {
4955 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4956 dev_priv->is_ddr3,
4957 dev_priv->fsb_freq,
4958 dev_priv->mem_freq)) {
4959 drm_info(&dev_priv->drm,
4960 "failed to find known CxSR latency "
4961 "(found ddr%s fsb freq %d, mem freq %d), "
4962 "disabling CxSR\n",
4963 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4964 dev_priv->fsb_freq, dev_priv->mem_freq);
4965 /* Disable CxSR and never update its watermark again */
4966 intel_set_memory_cxsr(dev_priv, false);
4967 dev_priv->display.funcs.wm = &nop_funcs;
4968 } else
4969 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4970 } else if (DISPLAY_VER(dev_priv) == 4) {
4971 dev_priv->display.funcs.wm = &i965_wm_funcs;
4972 } else if (DISPLAY_VER(dev_priv) == 3) {
4973 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4974 } else if (DISPLAY_VER(dev_priv) == 2) {
4975 if (INTEL_NUM_PIPES(dev_priv) == 1)
4976 dev_priv->display.funcs.wm = &i845_wm_funcs;
4977 else
4978 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4979 } else {
4980 drm_err(&dev_priv->drm,
4981 "unexpected fall-through in %s\n", __func__);
4982 dev_priv->display.funcs.wm = &nop_funcs;
4983 }
4984}
4985
4986void intel_pm_setup(struct drm_i915_private *dev_priv)
4987{
4988 dev_priv->runtime_pm.suspended = false;
4989 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
4990}
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include <linux/module.h>
29#include <linux/pm_runtime.h>
30
31#include <drm/drm_atomic_helper.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_plane_helper.h>
34
35#include "display/intel_atomic.h"
36#include "display/intel_atomic_plane.h"
37#include "display/intel_bw.h"
38#include "display/intel_de.h"
39#include "display/intel_display_types.h"
40#include "display/intel_fbc.h"
41#include "display/intel_sprite.h"
42#include "display/skl_universal_plane.h"
43
44#include "gt/intel_llc.h"
45
46#include "i915_drv.h"
47#include "i915_fixed.h"
48#include "i915_irq.h"
49#include "i915_trace.h"
50#include "intel_pm.h"
51#include "intel_sideband.h"
52#include "../../../platform/x86/intel_ips.h"
53
54/* Stores plane specific WM parameters */
55struct skl_wm_params {
56 bool x_tiled, y_tiled;
57 bool rc_surface;
58 bool is_planar;
59 u32 width;
60 u8 cpp;
61 u32 plane_pixel_rate;
62 u32 y_min_scanlines;
63 u32 plane_bytes_per_line;
64 uint_fixed_16_16_t plane_blocks_per_line;
65 uint_fixed_16_16_t y_tile_minimum;
66 u32 linetime_us;
67 u32 dbuf_block_size;
68};
69
70/* used in computing the new watermarks state */
71struct intel_wm_config {
72 unsigned int num_pipes_active;
73 bool sprites_enabled;
74 bool sprites_scaled;
75};
76
77static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
78{
79 enum pipe pipe;
80
81 if (HAS_LLC(dev_priv)) {
82 /*
83 * WaCompressedResourceDisplayNewHashMode:skl,kbl
84 * Display WA #0390: skl,kbl
85 *
86 * Must match Sampler, Pixel Back End, and Media. See
87 * WaCompressedResourceSamplerPbeMediaNewHashMode.
88 */
89 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
90 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
91 SKL_DE_COMPRESSED_HASH_MODE);
92 }
93
94 for_each_pipe(dev_priv, pipe) {
95 /*
96 * "Plane N strech max must be programmed to 11b (x1)
97 * when Async flips are enabled on that plane."
98 */
99 if (!IS_GEMINILAKE(dev_priv) && intel_vtd_active())
100 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
101 SKL_PLANE1_STRETCH_MAX_MASK, SKL_PLANE1_STRETCH_MAX_X1);
102 }
103
104 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
105 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
106 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
107
108 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
109 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
110 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
111
112 /*
113 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
114 * Display WA #0859: skl,bxt,kbl,glk,cfl
115 */
116 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
117 DISP_FBC_MEMORY_WAKE);
118}
119
120static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124 /* WaDisableSDEUnitClockGating:bxt */
125 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
126 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
127
128 /*
129 * FIXME:
130 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
131 */
132 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
133 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
134
135 /*
136 * Wa: Backlight PWM may stop in the asserted state, causing backlight
137 * to stay fully on.
138 */
139 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
140 PWM1_GATING_DIS | PWM2_GATING_DIS);
141
142 /*
143 * Lower the display internal timeout.
144 * This is needed to avoid any hard hangs when DSI port PLL
145 * is off and a MMIO access is attempted by any privilege
146 * application, using batch buffers or any other means.
147 */
148 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
149
150 /*
151 * WaFbcTurnOffFbcWatermark:bxt
152 * Display WA #0562: bxt
153 */
154 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
155 DISP_FBC_WM_DIS);
156
157 /*
158 * WaFbcHighMemBwCorruptionAvoidance:bxt
159 * Display WA #0883: bxt
160 */
161 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
162 ILK_DPFC_DISABLE_DUMMY0);
163}
164
165static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
166{
167 gen9_init_clock_gating(dev_priv);
168
169 /*
170 * WaDisablePWMClockGating:glk
171 * Backlight PWM may stop in the asserted state, causing backlight
172 * to stay fully on.
173 */
174 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
175 PWM1_GATING_DIS | PWM2_GATING_DIS);
176}
177
178static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
179{
180 u32 tmp;
181
182 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
183
184 switch (tmp & CLKCFG_FSB_MASK) {
185 case CLKCFG_FSB_533:
186 dev_priv->fsb_freq = 533; /* 133*4 */
187 break;
188 case CLKCFG_FSB_800:
189 dev_priv->fsb_freq = 800; /* 200*4 */
190 break;
191 case CLKCFG_FSB_667:
192 dev_priv->fsb_freq = 667; /* 167*4 */
193 break;
194 case CLKCFG_FSB_400:
195 dev_priv->fsb_freq = 400; /* 100*4 */
196 break;
197 }
198
199 switch (tmp & CLKCFG_MEM_MASK) {
200 case CLKCFG_MEM_533:
201 dev_priv->mem_freq = 533;
202 break;
203 case CLKCFG_MEM_667:
204 dev_priv->mem_freq = 667;
205 break;
206 case CLKCFG_MEM_800:
207 dev_priv->mem_freq = 800;
208 break;
209 }
210
211 /* detect pineview DDR3 setting */
212 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
213 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
214}
215
216static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
217{
218 u16 ddrpll, csipll;
219
220 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
221 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
222
223 switch (ddrpll & 0xff) {
224 case 0xc:
225 dev_priv->mem_freq = 800;
226 break;
227 case 0x10:
228 dev_priv->mem_freq = 1066;
229 break;
230 case 0x14:
231 dev_priv->mem_freq = 1333;
232 break;
233 case 0x18:
234 dev_priv->mem_freq = 1600;
235 break;
236 default:
237 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
238 ddrpll & 0xff);
239 dev_priv->mem_freq = 0;
240 break;
241 }
242
243 switch (csipll & 0x3ff) {
244 case 0x00c:
245 dev_priv->fsb_freq = 3200;
246 break;
247 case 0x00e:
248 dev_priv->fsb_freq = 3733;
249 break;
250 case 0x010:
251 dev_priv->fsb_freq = 4266;
252 break;
253 case 0x012:
254 dev_priv->fsb_freq = 4800;
255 break;
256 case 0x014:
257 dev_priv->fsb_freq = 5333;
258 break;
259 case 0x016:
260 dev_priv->fsb_freq = 5866;
261 break;
262 case 0x018:
263 dev_priv->fsb_freq = 6400;
264 break;
265 default:
266 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
267 csipll & 0x3ff);
268 dev_priv->fsb_freq = 0;
269 break;
270 }
271}
272
273static const struct cxsr_latency cxsr_latency_table[] = {
274 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
275 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
276 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
277 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
278 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
279
280 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
281 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
282 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
283 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
284 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
285
286 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
287 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
288 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
289 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
290 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
291
292 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
293 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
294 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
295 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
296 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
297
298 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
299 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
300 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
301 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
302 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
303
304 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
305 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
306 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
307 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
308 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
309};
310
311static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
312 bool is_ddr3,
313 int fsb,
314 int mem)
315{
316 const struct cxsr_latency *latency;
317 int i;
318
319 if (fsb == 0 || mem == 0)
320 return NULL;
321
322 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
323 latency = &cxsr_latency_table[i];
324 if (is_desktop == latency->is_desktop &&
325 is_ddr3 == latency->is_ddr3 &&
326 fsb == latency->fsb_freq && mem == latency->mem_freq)
327 return latency;
328 }
329
330 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
331
332 return NULL;
333}
334
335static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
336{
337 u32 val;
338
339 vlv_punit_get(dev_priv);
340
341 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
342 if (enable)
343 val &= ~FORCE_DDR_HIGH_FREQ;
344 else
345 val |= FORCE_DDR_HIGH_FREQ;
346 val &= ~FORCE_DDR_LOW_FREQ;
347 val |= FORCE_DDR_FREQ_REQ_ACK;
348 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
349
350 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
351 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
352 drm_err(&dev_priv->drm,
353 "timed out waiting for Punit DDR DVFS request\n");
354
355 vlv_punit_put(dev_priv);
356}
357
358static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
359{
360 u32 val;
361
362 vlv_punit_get(dev_priv);
363
364 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
365 if (enable)
366 val |= DSP_MAXFIFO_PM5_ENABLE;
367 else
368 val &= ~DSP_MAXFIFO_PM5_ENABLE;
369 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
370
371 vlv_punit_put(dev_priv);
372}
373
374#define FW_WM(value, plane) \
375 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
376
377static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
378{
379 bool was_enabled;
380 u32 val;
381
382 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
383 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
384 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
385 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
386 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
387 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
388 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
389 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
390 } else if (IS_PINEVIEW(dev_priv)) {
391 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
392 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
393 if (enable)
394 val |= PINEVIEW_SELF_REFRESH_EN;
395 else
396 val &= ~PINEVIEW_SELF_REFRESH_EN;
397 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
398 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
399 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
400 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
401 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
402 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
403 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
404 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
405 } else if (IS_I915GM(dev_priv)) {
406 /*
407 * FIXME can't find a bit like this for 915G, and
408 * and yet it does have the related watermark in
409 * FW_BLC_SELF. What's going on?
410 */
411 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
412 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
413 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
414 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
415 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
416 } else {
417 return false;
418 }
419
420 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
421
422 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
423 enableddisabled(enable),
424 enableddisabled(was_enabled));
425
426 return was_enabled;
427}
428
429/**
430 * intel_set_memory_cxsr - Configure CxSR state
431 * @dev_priv: i915 device
432 * @enable: Allow vs. disallow CxSR
433 *
434 * Allow or disallow the system to enter a special CxSR
435 * (C-state self refresh) state. What typically happens in CxSR mode
436 * is that several display FIFOs may get combined into a single larger
437 * FIFO for a particular plane (so called max FIFO mode) to allow the
438 * system to defer memory fetches longer, and the memory will enter
439 * self refresh.
440 *
441 * Note that enabling CxSR does not guarantee that the system enter
442 * this special mode, nor does it guarantee that the system stays
443 * in that mode once entered. So this just allows/disallows the system
444 * to autonomously utilize the CxSR mode. Other factors such as core
445 * C-states will affect when/if the system actually enters/exits the
446 * CxSR mode.
447 *
448 * Note that on VLV/CHV this actually only controls the max FIFO mode,
449 * and the system is free to enter/exit memory self refresh at any time
450 * even when the use of CxSR has been disallowed.
451 *
452 * While the system is actually in the CxSR/max FIFO mode, some plane
453 * control registers will not get latched on vblank. Thus in order to
454 * guarantee the system will respond to changes in the plane registers
455 * we must always disallow CxSR prior to making changes to those registers.
456 * Unfortunately the system will re-evaluate the CxSR conditions at
457 * frame start which happens after vblank start (which is when the plane
458 * registers would get latched), so we can't proceed with the plane update
459 * during the same frame where we disallowed CxSR.
460 *
461 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
462 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
463 * the hardware w.r.t. HPLL SR when writing to plane registers.
464 * Disallowing just CxSR is sufficient.
465 */
466bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
467{
468 bool ret;
469
470 mutex_lock(&dev_priv->wm.wm_mutex);
471 ret = _intel_set_memory_cxsr(dev_priv, enable);
472 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
473 dev_priv->wm.vlv.cxsr = enable;
474 else if (IS_G4X(dev_priv))
475 dev_priv->wm.g4x.cxsr = enable;
476 mutex_unlock(&dev_priv->wm.wm_mutex);
477
478 return ret;
479}
480
481/*
482 * Latency for FIFO fetches is dependent on several factors:
483 * - memory configuration (speed, channels)
484 * - chipset
485 * - current MCH state
486 * It can be fairly high in some situations, so here we assume a fairly
487 * pessimal value. It's a tradeoff between extra memory fetches (if we
488 * set this value too high, the FIFO will fetch frequently to stay full)
489 * and power consumption (set it too low to save power and we might see
490 * FIFO underruns and display "flicker").
491 *
492 * A value of 5us seems to be a good balance; safe for very low end
493 * platforms but not overly aggressive on lower latency configs.
494 */
495static const int pessimal_latency_ns = 5000;
496
497#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
498 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
499
500static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
501{
502 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
503 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
504 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
505 enum pipe pipe = crtc->pipe;
506 int sprite0_start, sprite1_start;
507 u32 dsparb, dsparb2, dsparb3;
508
509 switch (pipe) {
510 case PIPE_A:
511 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
512 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
513 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
514 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
515 break;
516 case PIPE_B:
517 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
518 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
519 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
520 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
521 break;
522 case PIPE_C:
523 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
524 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
525 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
526 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
527 break;
528 default:
529 MISSING_CASE(pipe);
530 return;
531 }
532
533 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
534 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
535 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
536 fifo_state->plane[PLANE_CURSOR] = 63;
537}
538
539static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
540 enum i9xx_plane_id i9xx_plane)
541{
542 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
543 int size;
544
545 size = dsparb & 0x7f;
546 if (i9xx_plane == PLANE_B)
547 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
548
549 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
550 dsparb, plane_name(i9xx_plane), size);
551
552 return size;
553}
554
555static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
556 enum i9xx_plane_id i9xx_plane)
557{
558 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
559 int size;
560
561 size = dsparb & 0x1ff;
562 if (i9xx_plane == PLANE_B)
563 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
564 size >>= 1; /* Convert to cachelines */
565
566 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
567 dsparb, plane_name(i9xx_plane), size);
568
569 return size;
570}
571
572static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
573 enum i9xx_plane_id i9xx_plane)
574{
575 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
576 int size;
577
578 size = dsparb & 0x7f;
579 size >>= 2; /* Convert to cachelines */
580
581 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
582 dsparb, plane_name(i9xx_plane), size);
583
584 return size;
585}
586
587/* Pineview has different values for various configs */
588static const struct intel_watermark_params pnv_display_wm = {
589 .fifo_size = PINEVIEW_DISPLAY_FIFO,
590 .max_wm = PINEVIEW_MAX_WM,
591 .default_wm = PINEVIEW_DFT_WM,
592 .guard_size = PINEVIEW_GUARD_WM,
593 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594};
595
596static const struct intel_watermark_params pnv_display_hplloff_wm = {
597 .fifo_size = PINEVIEW_DISPLAY_FIFO,
598 .max_wm = PINEVIEW_MAX_WM,
599 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
600 .guard_size = PINEVIEW_GUARD_WM,
601 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
602};
603
604static const struct intel_watermark_params pnv_cursor_wm = {
605 .fifo_size = PINEVIEW_CURSOR_FIFO,
606 .max_wm = PINEVIEW_CURSOR_MAX_WM,
607 .default_wm = PINEVIEW_CURSOR_DFT_WM,
608 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
609 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
610};
611
612static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
613 .fifo_size = PINEVIEW_CURSOR_FIFO,
614 .max_wm = PINEVIEW_CURSOR_MAX_WM,
615 .default_wm = PINEVIEW_CURSOR_DFT_WM,
616 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
617 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
618};
619
620static const struct intel_watermark_params i965_cursor_wm_info = {
621 .fifo_size = I965_CURSOR_FIFO,
622 .max_wm = I965_CURSOR_MAX_WM,
623 .default_wm = I965_CURSOR_DFT_WM,
624 .guard_size = 2,
625 .cacheline_size = I915_FIFO_LINE_SIZE,
626};
627
628static const struct intel_watermark_params i945_wm_info = {
629 .fifo_size = I945_FIFO_SIZE,
630 .max_wm = I915_MAX_WM,
631 .default_wm = 1,
632 .guard_size = 2,
633 .cacheline_size = I915_FIFO_LINE_SIZE,
634};
635
636static const struct intel_watermark_params i915_wm_info = {
637 .fifo_size = I915_FIFO_SIZE,
638 .max_wm = I915_MAX_WM,
639 .default_wm = 1,
640 .guard_size = 2,
641 .cacheline_size = I915_FIFO_LINE_SIZE,
642};
643
644static const struct intel_watermark_params i830_a_wm_info = {
645 .fifo_size = I855GM_FIFO_SIZE,
646 .max_wm = I915_MAX_WM,
647 .default_wm = 1,
648 .guard_size = 2,
649 .cacheline_size = I830_FIFO_LINE_SIZE,
650};
651
652static const struct intel_watermark_params i830_bc_wm_info = {
653 .fifo_size = I855GM_FIFO_SIZE,
654 .max_wm = I915_MAX_WM/2,
655 .default_wm = 1,
656 .guard_size = 2,
657 .cacheline_size = I830_FIFO_LINE_SIZE,
658};
659
660static const struct intel_watermark_params i845_wm_info = {
661 .fifo_size = I830_FIFO_SIZE,
662 .max_wm = I915_MAX_WM,
663 .default_wm = 1,
664 .guard_size = 2,
665 .cacheline_size = I830_FIFO_LINE_SIZE,
666};
667
668/**
669 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @cpp: Plane bytes per pixel
672 * @latency: Memory wakeup latency in 0.1us units
673 *
674 * Compute the watermark using the method 1 or "small buffer"
675 * formula. The caller may additonally add extra cachelines
676 * to account for TLB misses and clock crossings.
677 *
678 * This method is concerned with the short term drain rate
679 * of the FIFO, ie. it does not account for blanking periods
680 * which would effectively reduce the average drain rate across
681 * a longer period. The name "small" refers to the fact the
682 * FIFO is relatively small compared to the amount of data
683 * fetched.
684 *
685 * The FIFO level vs. time graph might look something like:
686 *
687 * |\ |\
688 * | \ | \
689 * __---__---__ (- plane active, _ blanking)
690 * -> time
691 *
692 * or perhaps like this:
693 *
694 * |\|\ |\|\
695 * __----__----__ (- plane active, _ blanking)
696 * -> time
697 *
698 * Returns:
699 * The watermark in bytes
700 */
701static unsigned int intel_wm_method1(unsigned int pixel_rate,
702 unsigned int cpp,
703 unsigned int latency)
704{
705 u64 ret;
706
707 ret = mul_u32_u32(pixel_rate, cpp * latency);
708 ret = DIV_ROUND_UP_ULL(ret, 10000);
709
710 return ret;
711}
712
713/**
714 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
715 * @pixel_rate: Pipe pixel rate in kHz
716 * @htotal: Pipe horizontal total
717 * @width: Plane width in pixels
718 * @cpp: Plane bytes per pixel
719 * @latency: Memory wakeup latency in 0.1us units
720 *
721 * Compute the watermark using the method 2 or "large buffer"
722 * formula. The caller may additonally add extra cachelines
723 * to account for TLB misses and clock crossings.
724 *
725 * This method is concerned with the long term drain rate
726 * of the FIFO, ie. it does account for blanking periods
727 * which effectively reduce the average drain rate across
728 * a longer period. The name "large" refers to the fact the
729 * FIFO is relatively large compared to the amount of data
730 * fetched.
731 *
732 * The FIFO level vs. time graph might look something like:
733 *
734 * |\___ |\___
735 * | \___ | \___
736 * | \ | \
737 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
738 * -> time
739 *
740 * Returns:
741 * The watermark in bytes
742 */
743static unsigned int intel_wm_method2(unsigned int pixel_rate,
744 unsigned int htotal,
745 unsigned int width,
746 unsigned int cpp,
747 unsigned int latency)
748{
749 unsigned int ret;
750
751 /*
752 * FIXME remove once all users are computing
753 * watermarks in the correct place.
754 */
755 if (WARN_ON_ONCE(htotal == 0))
756 htotal = 1;
757
758 ret = (latency * pixel_rate) / (htotal * 10000);
759 ret = (ret + 1) * width * cpp;
760
761 return ret;
762}
763
764/**
765 * intel_calculate_wm - calculate watermark level
766 * @pixel_rate: pixel clock
767 * @wm: chip FIFO params
768 * @fifo_size: size of the FIFO buffer
769 * @cpp: bytes per pixel
770 * @latency_ns: memory latency for the platform
771 *
772 * Calculate the watermark level (the level at which the display plane will
773 * start fetching from memory again). Each chip has a different display
774 * FIFO size and allocation, so the caller needs to figure that out and pass
775 * in the correct intel_watermark_params structure.
776 *
777 * As the pixel clock runs, the FIFO will be drained at a rate that depends
778 * on the pixel size. When it reaches the watermark level, it'll start
779 * fetching FIFO line sized based chunks from memory until the FIFO fills
780 * past the watermark point. If the FIFO drains completely, a FIFO underrun
781 * will occur, and a display engine hang could result.
782 */
783static unsigned int intel_calculate_wm(int pixel_rate,
784 const struct intel_watermark_params *wm,
785 int fifo_size, int cpp,
786 unsigned int latency_ns)
787{
788 int entries, wm_size;
789
790 /*
791 * Note: we need to make sure we don't overflow for various clock &
792 * latency values.
793 * clocks go from a few thousand to several hundred thousand.
794 * latency is usually a few thousand
795 */
796 entries = intel_wm_method1(pixel_rate, cpp,
797 latency_ns / 100);
798 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
799 wm->guard_size;
800 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
801
802 wm_size = fifo_size - entries;
803 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
804
805 /* Don't promote wm_size to unsigned... */
806 if (wm_size > wm->max_wm)
807 wm_size = wm->max_wm;
808 if (wm_size <= 0)
809 wm_size = wm->default_wm;
810
811 /*
812 * Bspec seems to indicate that the value shouldn't be lower than
813 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
814 * Lets go for 8 which is the burst size since certain platforms
815 * already use a hardcoded 8 (which is what the spec says should be
816 * done).
817 */
818 if (wm_size <= 8)
819 wm_size = 8;
820
821 return wm_size;
822}
823
824static bool is_disabling(int old, int new, int threshold)
825{
826 return old >= threshold && new < threshold;
827}
828
829static bool is_enabling(int old, int new, int threshold)
830{
831 return old < threshold && new >= threshold;
832}
833
834static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
835{
836 return dev_priv->wm.max_level + 1;
837}
838
839static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
840 const struct intel_plane_state *plane_state)
841{
842 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
843
844 /* FIXME check the 'enable' instead */
845 if (!crtc_state->hw.active)
846 return false;
847
848 /*
849 * Treat cursor with fb as always visible since cursor updates
850 * can happen faster than the vrefresh rate, and the current
851 * watermark code doesn't handle that correctly. Cursor updates
852 * which set/clear the fb or change the cursor size are going
853 * to get throttled by intel_legacy_cursor_update() to work
854 * around this problem with the watermark code.
855 */
856 if (plane->id == PLANE_CURSOR)
857 return plane_state->hw.fb != NULL;
858 else
859 return plane_state->uapi.visible;
860}
861
862static bool intel_crtc_active(struct intel_crtc *crtc)
863{
864 /* Be paranoid as we can arrive here with only partial
865 * state retrieved from the hardware during setup.
866 *
867 * We can ditch the adjusted_mode.crtc_clock check as soon
868 * as Haswell has gained clock readout/fastboot support.
869 *
870 * We can ditch the crtc->primary->state->fb check as soon as we can
871 * properly reconstruct framebuffers.
872 *
873 * FIXME: The intel_crtc->active here should be switched to
874 * crtc->state->active once we have proper CRTC states wired up
875 * for atomic.
876 */
877 return crtc->active && crtc->base.primary->state->fb &&
878 crtc->config->hw.adjusted_mode.crtc_clock;
879}
880
881static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
882{
883 struct intel_crtc *crtc, *enabled = NULL;
884
885 for_each_intel_crtc(&dev_priv->drm, crtc) {
886 if (intel_crtc_active(crtc)) {
887 if (enabled)
888 return NULL;
889 enabled = crtc;
890 }
891 }
892
893 return enabled;
894}
895
896static void pnv_update_wm(struct intel_crtc *unused_crtc)
897{
898 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
899 struct intel_crtc *crtc;
900 const struct cxsr_latency *latency;
901 u32 reg;
902 unsigned int wm;
903
904 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
905 dev_priv->is_ddr3,
906 dev_priv->fsb_freq,
907 dev_priv->mem_freq);
908 if (!latency) {
909 drm_dbg_kms(&dev_priv->drm,
910 "Unknown FSB/MEM found, disable CxSR\n");
911 intel_set_memory_cxsr(dev_priv, false);
912 return;
913 }
914
915 crtc = single_enabled_crtc(dev_priv);
916 if (crtc) {
917 const struct drm_display_mode *pipe_mode =
918 &crtc->config->hw.pipe_mode;
919 const struct drm_framebuffer *fb =
920 crtc->base.primary->state->fb;
921 int cpp = fb->format->cpp[0];
922 int clock = pipe_mode->crtc_clock;
923
924 /* Display SR */
925 wm = intel_calculate_wm(clock, &pnv_display_wm,
926 pnv_display_wm.fifo_size,
927 cpp, latency->display_sr);
928 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
929 reg &= ~DSPFW_SR_MASK;
930 reg |= FW_WM(wm, SR);
931 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
932 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
933
934 /* cursor SR */
935 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
936 pnv_display_wm.fifo_size,
937 4, latency->cursor_sr);
938 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
939 reg &= ~DSPFW_CURSOR_SR_MASK;
940 reg |= FW_WM(wm, CURSOR_SR);
941 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
942
943 /* Display HPLL off SR */
944 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 cpp, latency->display_hpll_disable);
947 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
948 reg &= ~DSPFW_HPLL_SR_MASK;
949 reg |= FW_WM(wm, HPLL_SR);
950 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
951
952 /* cursor HPLL off SR */
953 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
954 pnv_display_hplloff_wm.fifo_size,
955 4, latency->cursor_hpll_disable);
956 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
957 reg &= ~DSPFW_HPLL_CURSOR_MASK;
958 reg |= FW_WM(wm, HPLL_CURSOR);
959 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
960 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
961
962 intel_set_memory_cxsr(dev_priv, true);
963 } else {
964 intel_set_memory_cxsr(dev_priv, false);
965 }
966}
967
968/*
969 * Documentation says:
970 * "If the line size is small, the TLB fetches can get in the way of the
971 * data fetches, causing some lag in the pixel data return which is not
972 * accounted for in the above formulas. The following adjustment only
973 * needs to be applied if eight whole lines fit in the buffer at once.
974 * The WM is adjusted upwards by the difference between the FIFO size
975 * and the size of 8 whole lines. This adjustment is always performed
976 * in the actual pixel depth regardless of whether FBC is enabled or not."
977 */
978static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
979{
980 int tlb_miss = fifo_size * 64 - width * cpp * 8;
981
982 return max(0, tlb_miss);
983}
984
985static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
986 const struct g4x_wm_values *wm)
987{
988 enum pipe pipe;
989
990 for_each_pipe(dev_priv, pipe)
991 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
992
993 intel_uncore_write(&dev_priv->uncore, DSPFW1,
994 FW_WM(wm->sr.plane, SR) |
995 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
996 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
998 intel_uncore_write(&dev_priv->uncore, DSPFW2,
999 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
1000 FW_WM(wm->sr.fbc, FBC_SR) |
1001 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1002 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1003 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1004 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1005 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1006 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1007 FW_WM(wm->sr.cursor, CURSOR_SR) |
1008 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1009 FW_WM(wm->hpll.plane, HPLL_SR));
1010
1011 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1012}
1013
1014#define FW_WM_VLV(value, plane) \
1015 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1016
1017static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1018 const struct vlv_wm_values *wm)
1019{
1020 enum pipe pipe;
1021
1022 for_each_pipe(dev_priv, pipe) {
1023 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1024
1025 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1026 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1027 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1028 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1029 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1030 }
1031
1032 /*
1033 * Zero the (unused) WM1 watermarks, and also clear all the
1034 * high order bits so that there are no out of bounds values
1035 * present in the registers during the reprogramming.
1036 */
1037 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1039 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1040 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1041 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1042
1043 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1044 FW_WM(wm->sr.plane, SR) |
1045 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1046 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1047 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1048 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1049 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1050 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1051 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1052 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1053 FW_WM(wm->sr.cursor, CURSOR_SR));
1054
1055 if (IS_CHERRYVIEW(dev_priv)) {
1056 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1057 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1058 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1059 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1060 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1061 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1062 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1063 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1065 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1066 FW_WM(wm->sr.plane >> 9, SR_HI) |
1067 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1068 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1069 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1072 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1075 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1076 } else {
1077 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1078 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1079 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1080 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1081 FW_WM(wm->sr.plane >> 9, SR_HI) |
1082 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1083 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1084 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1085 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1086 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1087 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1088 }
1089
1090 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1091}
1092
1093#undef FW_WM_VLV
1094
1095static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1096{
1097 /* all latencies in usec */
1098 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1099 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1100 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1101
1102 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1103}
1104
1105static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1106{
1107 /*
1108 * DSPCNTR[13] supposedly controls whether the
1109 * primary plane can use the FIFO space otherwise
1110 * reserved for the sprite plane. It's not 100% clear
1111 * what the actual FIFO size is, but it looks like we
1112 * can happily set both primary and sprite watermarks
1113 * up to 127 cachelines. So that would seem to mean
1114 * that either DSPCNTR[13] doesn't do anything, or that
1115 * the total FIFO is >= 256 cachelines in size. Either
1116 * way, we don't seem to have to worry about this
1117 * repartitioning as the maximum watermark value the
1118 * register can hold for each plane is lower than the
1119 * minimum FIFO size.
1120 */
1121 switch (plane_id) {
1122 case PLANE_CURSOR:
1123 return 63;
1124 case PLANE_PRIMARY:
1125 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1126 case PLANE_SPRITE0:
1127 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1128 default:
1129 MISSING_CASE(plane_id);
1130 return 0;
1131 }
1132}
1133
1134static int g4x_fbc_fifo_size(int level)
1135{
1136 switch (level) {
1137 case G4X_WM_LEVEL_SR:
1138 return 7;
1139 case G4X_WM_LEVEL_HPLL:
1140 return 15;
1141 default:
1142 MISSING_CASE(level);
1143 return 0;
1144 }
1145}
1146
1147static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1148 const struct intel_plane_state *plane_state,
1149 int level)
1150{
1151 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1152 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1153 const struct drm_display_mode *pipe_mode =
1154 &crtc_state->hw.pipe_mode;
1155 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1156 unsigned int clock, htotal, cpp, width, wm;
1157
1158 if (latency == 0)
1159 return USHRT_MAX;
1160
1161 if (!intel_wm_plane_visible(crtc_state, plane_state))
1162 return 0;
1163
1164 cpp = plane_state->hw.fb->format->cpp[0];
1165
1166 /*
1167 * Not 100% sure which way ELK should go here as the
1168 * spec only says CL/CTG should assume 32bpp and BW
1169 * doesn't need to. But as these things followed the
1170 * mobile vs. desktop lines on gen3 as well, let's
1171 * assume ELK doesn't need this.
1172 *
1173 * The spec also fails to list such a restriction for
1174 * the HPLL watermark, which seems a little strange.
1175 * Let's use 32bpp for the HPLL watermark as well.
1176 */
1177 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1178 level != G4X_WM_LEVEL_NORMAL)
1179 cpp = max(cpp, 4u);
1180
1181 clock = pipe_mode->crtc_clock;
1182 htotal = pipe_mode->crtc_htotal;
1183
1184 width = drm_rect_width(&plane_state->uapi.dst);
1185
1186 if (plane->id == PLANE_CURSOR) {
1187 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1188 } else if (plane->id == PLANE_PRIMARY &&
1189 level == G4X_WM_LEVEL_NORMAL) {
1190 wm = intel_wm_method1(clock, cpp, latency);
1191 } else {
1192 unsigned int small, large;
1193
1194 small = intel_wm_method1(clock, cpp, latency);
1195 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1196
1197 wm = min(small, large);
1198 }
1199
1200 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1201 width, cpp);
1202
1203 wm = DIV_ROUND_UP(wm, 64) + 2;
1204
1205 return min_t(unsigned int, wm, USHRT_MAX);
1206}
1207
1208static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1209 int level, enum plane_id plane_id, u16 value)
1210{
1211 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1212 bool dirty = false;
1213
1214 for (; level < intel_wm_num_levels(dev_priv); level++) {
1215 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1216
1217 dirty |= raw->plane[plane_id] != value;
1218 raw->plane[plane_id] = value;
1219 }
1220
1221 return dirty;
1222}
1223
1224static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1225 int level, u16 value)
1226{
1227 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1228 bool dirty = false;
1229
1230 /* NORMAL level doesn't have an FBC watermark */
1231 level = max(level, G4X_WM_LEVEL_SR);
1232
1233 for (; level < intel_wm_num_levels(dev_priv); level++) {
1234 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1235
1236 dirty |= raw->fbc != value;
1237 raw->fbc = value;
1238 }
1239
1240 return dirty;
1241}
1242
1243static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1244 const struct intel_plane_state *plane_state,
1245 u32 pri_val);
1246
1247static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1248 const struct intel_plane_state *plane_state)
1249{
1250 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1251 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1252 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1253 enum plane_id plane_id = plane->id;
1254 bool dirty = false;
1255 int level;
1256
1257 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1258 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1259 if (plane_id == PLANE_PRIMARY)
1260 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1261 goto out;
1262 }
1263
1264 for (level = 0; level < num_levels; level++) {
1265 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1266 int wm, max_wm;
1267
1268 wm = g4x_compute_wm(crtc_state, plane_state, level);
1269 max_wm = g4x_plane_fifo_size(plane_id, level);
1270
1271 if (wm > max_wm)
1272 break;
1273
1274 dirty |= raw->plane[plane_id] != wm;
1275 raw->plane[plane_id] = wm;
1276
1277 if (plane_id != PLANE_PRIMARY ||
1278 level == G4X_WM_LEVEL_NORMAL)
1279 continue;
1280
1281 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1282 raw->plane[plane_id]);
1283 max_wm = g4x_fbc_fifo_size(level);
1284
1285 /*
1286 * FBC wm is not mandatory as we
1287 * can always just disable its use.
1288 */
1289 if (wm > max_wm)
1290 wm = USHRT_MAX;
1291
1292 dirty |= raw->fbc != wm;
1293 raw->fbc = wm;
1294 }
1295
1296 /* mark watermarks as invalid */
1297 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1298
1299 if (plane_id == PLANE_PRIMARY)
1300 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1301
1302 out:
1303 if (dirty) {
1304 drm_dbg_kms(&dev_priv->drm,
1305 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1306 plane->base.name,
1307 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1308 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1309 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1310
1311 if (plane_id == PLANE_PRIMARY)
1312 drm_dbg_kms(&dev_priv->drm,
1313 "FBC watermarks: SR=%d, HPLL=%d\n",
1314 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1315 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1316 }
1317
1318 return dirty;
1319}
1320
1321static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1322 enum plane_id plane_id, int level)
1323{
1324 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1325
1326 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1327}
1328
1329static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1330 int level)
1331{
1332 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1333
1334 if (level > dev_priv->wm.max_level)
1335 return false;
1336
1337 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1338 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1339 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1340}
1341
1342/* mark all levels starting from 'level' as invalid */
1343static void g4x_invalidate_wms(struct intel_crtc *crtc,
1344 struct g4x_wm_state *wm_state, int level)
1345{
1346 if (level <= G4X_WM_LEVEL_NORMAL) {
1347 enum plane_id plane_id;
1348
1349 for_each_plane_id_on_crtc(crtc, plane_id)
1350 wm_state->wm.plane[plane_id] = USHRT_MAX;
1351 }
1352
1353 if (level <= G4X_WM_LEVEL_SR) {
1354 wm_state->cxsr = false;
1355 wm_state->sr.cursor = USHRT_MAX;
1356 wm_state->sr.plane = USHRT_MAX;
1357 wm_state->sr.fbc = USHRT_MAX;
1358 }
1359
1360 if (level <= G4X_WM_LEVEL_HPLL) {
1361 wm_state->hpll_en = false;
1362 wm_state->hpll.cursor = USHRT_MAX;
1363 wm_state->hpll.plane = USHRT_MAX;
1364 wm_state->hpll.fbc = USHRT_MAX;
1365 }
1366}
1367
1368static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1369 int level)
1370{
1371 if (level < G4X_WM_LEVEL_SR)
1372 return false;
1373
1374 if (level >= G4X_WM_LEVEL_SR &&
1375 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1376 return false;
1377
1378 if (level >= G4X_WM_LEVEL_HPLL &&
1379 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1380 return false;
1381
1382 return true;
1383}
1384
1385static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1386{
1387 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1388 struct intel_atomic_state *state =
1389 to_intel_atomic_state(crtc_state->uapi.state);
1390 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1391 int num_active_planes = hweight8(crtc_state->active_planes &
1392 ~BIT(PLANE_CURSOR));
1393 const struct g4x_pipe_wm *raw;
1394 const struct intel_plane_state *old_plane_state;
1395 const struct intel_plane_state *new_plane_state;
1396 struct intel_plane *plane;
1397 enum plane_id plane_id;
1398 int i, level;
1399 unsigned int dirty = 0;
1400
1401 for_each_oldnew_intel_plane_in_state(state, plane,
1402 old_plane_state,
1403 new_plane_state, i) {
1404 if (new_plane_state->hw.crtc != &crtc->base &&
1405 old_plane_state->hw.crtc != &crtc->base)
1406 continue;
1407
1408 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1409 dirty |= BIT(plane->id);
1410 }
1411
1412 if (!dirty)
1413 return 0;
1414
1415 level = G4X_WM_LEVEL_NORMAL;
1416 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1417 goto out;
1418
1419 raw = &crtc_state->wm.g4x.raw[level];
1420 for_each_plane_id_on_crtc(crtc, plane_id)
1421 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1422
1423 level = G4X_WM_LEVEL_SR;
1424 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1425 goto out;
1426
1427 raw = &crtc_state->wm.g4x.raw[level];
1428 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1429 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1430 wm_state->sr.fbc = raw->fbc;
1431
1432 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1433
1434 level = G4X_WM_LEVEL_HPLL;
1435 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1436 goto out;
1437
1438 raw = &crtc_state->wm.g4x.raw[level];
1439 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1440 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1441 wm_state->hpll.fbc = raw->fbc;
1442
1443 wm_state->hpll_en = wm_state->cxsr;
1444
1445 level++;
1446
1447 out:
1448 if (level == G4X_WM_LEVEL_NORMAL)
1449 return -EINVAL;
1450
1451 /* invalidate the higher levels */
1452 g4x_invalidate_wms(crtc, wm_state, level);
1453
1454 /*
1455 * Determine if the FBC watermark(s) can be used. IF
1456 * this isn't the case we prefer to disable the FBC
1457 * watermark(s) rather than disable the SR/HPLL
1458 * level(s) entirely. 'level-1' is the highest valid
1459 * level here.
1460 */
1461 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1462
1463 return 0;
1464}
1465
1466static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1467{
1468 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1469 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1470 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1471 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1472 struct intel_atomic_state *intel_state =
1473 to_intel_atomic_state(new_crtc_state->uapi.state);
1474 const struct intel_crtc_state *old_crtc_state =
1475 intel_atomic_get_old_crtc_state(intel_state, crtc);
1476 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1477 enum plane_id plane_id;
1478
1479 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1480 *intermediate = *optimal;
1481
1482 intermediate->cxsr = false;
1483 intermediate->hpll_en = false;
1484 goto out;
1485 }
1486
1487 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1488 !new_crtc_state->disable_cxsr;
1489 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1490 !new_crtc_state->disable_cxsr;
1491 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1492
1493 for_each_plane_id_on_crtc(crtc, plane_id) {
1494 intermediate->wm.plane[plane_id] =
1495 max(optimal->wm.plane[plane_id],
1496 active->wm.plane[plane_id]);
1497
1498 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1499 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1500 }
1501
1502 intermediate->sr.plane = max(optimal->sr.plane,
1503 active->sr.plane);
1504 intermediate->sr.cursor = max(optimal->sr.cursor,
1505 active->sr.cursor);
1506 intermediate->sr.fbc = max(optimal->sr.fbc,
1507 active->sr.fbc);
1508
1509 intermediate->hpll.plane = max(optimal->hpll.plane,
1510 active->hpll.plane);
1511 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1512 active->hpll.cursor);
1513 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1514 active->hpll.fbc);
1515
1516 drm_WARN_ON(&dev_priv->drm,
1517 (intermediate->sr.plane >
1518 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1519 intermediate->sr.cursor >
1520 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1521 intermediate->cxsr);
1522 drm_WARN_ON(&dev_priv->drm,
1523 (intermediate->sr.plane >
1524 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1525 intermediate->sr.cursor >
1526 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1527 intermediate->hpll_en);
1528
1529 drm_WARN_ON(&dev_priv->drm,
1530 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1531 intermediate->fbc_en && intermediate->cxsr);
1532 drm_WARN_ON(&dev_priv->drm,
1533 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1534 intermediate->fbc_en && intermediate->hpll_en);
1535
1536out:
1537 /*
1538 * If our intermediate WM are identical to the final WM, then we can
1539 * omit the post-vblank programming; only update if it's different.
1540 */
1541 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1542 new_crtc_state->wm.need_postvbl_update = true;
1543
1544 return 0;
1545}
1546
1547static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1548 struct g4x_wm_values *wm)
1549{
1550 struct intel_crtc *crtc;
1551 int num_active_pipes = 0;
1552
1553 wm->cxsr = true;
1554 wm->hpll_en = true;
1555 wm->fbc_en = true;
1556
1557 for_each_intel_crtc(&dev_priv->drm, crtc) {
1558 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1559
1560 if (!crtc->active)
1561 continue;
1562
1563 if (!wm_state->cxsr)
1564 wm->cxsr = false;
1565 if (!wm_state->hpll_en)
1566 wm->hpll_en = false;
1567 if (!wm_state->fbc_en)
1568 wm->fbc_en = false;
1569
1570 num_active_pipes++;
1571 }
1572
1573 if (num_active_pipes != 1) {
1574 wm->cxsr = false;
1575 wm->hpll_en = false;
1576 wm->fbc_en = false;
1577 }
1578
1579 for_each_intel_crtc(&dev_priv->drm, crtc) {
1580 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1581 enum pipe pipe = crtc->pipe;
1582
1583 wm->pipe[pipe] = wm_state->wm;
1584 if (crtc->active && wm->cxsr)
1585 wm->sr = wm_state->sr;
1586 if (crtc->active && wm->hpll_en)
1587 wm->hpll = wm_state->hpll;
1588 }
1589}
1590
1591static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1592{
1593 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1594 struct g4x_wm_values new_wm = {};
1595
1596 g4x_merge_wm(dev_priv, &new_wm);
1597
1598 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1599 return;
1600
1601 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1602 _intel_set_memory_cxsr(dev_priv, false);
1603
1604 g4x_write_wm_values(dev_priv, &new_wm);
1605
1606 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1607 _intel_set_memory_cxsr(dev_priv, true);
1608
1609 *old_wm = new_wm;
1610}
1611
1612static void g4x_initial_watermarks(struct intel_atomic_state *state,
1613 struct intel_crtc *crtc)
1614{
1615 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1616 const struct intel_crtc_state *crtc_state =
1617 intel_atomic_get_new_crtc_state(state, crtc);
1618
1619 mutex_lock(&dev_priv->wm.wm_mutex);
1620 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1621 g4x_program_watermarks(dev_priv);
1622 mutex_unlock(&dev_priv->wm.wm_mutex);
1623}
1624
1625static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1626 struct intel_crtc *crtc)
1627{
1628 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1629 const struct intel_crtc_state *crtc_state =
1630 intel_atomic_get_new_crtc_state(state, crtc);
1631
1632 if (!crtc_state->wm.need_postvbl_update)
1633 return;
1634
1635 mutex_lock(&dev_priv->wm.wm_mutex);
1636 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1637 g4x_program_watermarks(dev_priv);
1638 mutex_unlock(&dev_priv->wm.wm_mutex);
1639}
1640
1641/* latency must be in 0.1us units. */
1642static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1643 unsigned int htotal,
1644 unsigned int width,
1645 unsigned int cpp,
1646 unsigned int latency)
1647{
1648 unsigned int ret;
1649
1650 ret = intel_wm_method2(pixel_rate, htotal,
1651 width, cpp, latency);
1652 ret = DIV_ROUND_UP(ret, 64);
1653
1654 return ret;
1655}
1656
1657static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1658{
1659 /* all latencies in usec */
1660 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1661
1662 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1663
1664 if (IS_CHERRYVIEW(dev_priv)) {
1665 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1666 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1667
1668 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1669 }
1670}
1671
1672static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1673 const struct intel_plane_state *plane_state,
1674 int level)
1675{
1676 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1677 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1678 const struct drm_display_mode *pipe_mode =
1679 &crtc_state->hw.pipe_mode;
1680 unsigned int clock, htotal, cpp, width, wm;
1681
1682 if (dev_priv->wm.pri_latency[level] == 0)
1683 return USHRT_MAX;
1684
1685 if (!intel_wm_plane_visible(crtc_state, plane_state))
1686 return 0;
1687
1688 cpp = plane_state->hw.fb->format->cpp[0];
1689 clock = pipe_mode->crtc_clock;
1690 htotal = pipe_mode->crtc_htotal;
1691 width = crtc_state->pipe_src_w;
1692
1693 if (plane->id == PLANE_CURSOR) {
1694 /*
1695 * FIXME the formula gives values that are
1696 * too big for the cursor FIFO, and hence we
1697 * would never be able to use cursors. For
1698 * now just hardcode the watermark.
1699 */
1700 wm = 63;
1701 } else {
1702 wm = vlv_wm_method2(clock, htotal, width, cpp,
1703 dev_priv->wm.pri_latency[level] * 10);
1704 }
1705
1706 return min_t(unsigned int, wm, USHRT_MAX);
1707}
1708
1709static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1710{
1711 return (active_planes & (BIT(PLANE_SPRITE0) |
1712 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1713}
1714
1715static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1716{
1717 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1718 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1719 const struct g4x_pipe_wm *raw =
1720 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1721 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1722 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1723 int num_active_planes = hweight8(active_planes);
1724 const int fifo_size = 511;
1725 int fifo_extra, fifo_left = fifo_size;
1726 int sprite0_fifo_extra = 0;
1727 unsigned int total_rate;
1728 enum plane_id plane_id;
1729
1730 /*
1731 * When enabling sprite0 after sprite1 has already been enabled
1732 * we tend to get an underrun unless sprite0 already has some
1733 * FIFO space allcoated. Hence we always allocate at least one
1734 * cacheline for sprite0 whenever sprite1 is enabled.
1735 *
1736 * All other plane enable sequences appear immune to this problem.
1737 */
1738 if (vlv_need_sprite0_fifo_workaround(active_planes))
1739 sprite0_fifo_extra = 1;
1740
1741 total_rate = raw->plane[PLANE_PRIMARY] +
1742 raw->plane[PLANE_SPRITE0] +
1743 raw->plane[PLANE_SPRITE1] +
1744 sprite0_fifo_extra;
1745
1746 if (total_rate > fifo_size)
1747 return -EINVAL;
1748
1749 if (total_rate == 0)
1750 total_rate = 1;
1751
1752 for_each_plane_id_on_crtc(crtc, plane_id) {
1753 unsigned int rate;
1754
1755 if ((active_planes & BIT(plane_id)) == 0) {
1756 fifo_state->plane[plane_id] = 0;
1757 continue;
1758 }
1759
1760 rate = raw->plane[plane_id];
1761 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1762 fifo_left -= fifo_state->plane[plane_id];
1763 }
1764
1765 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1766 fifo_left -= sprite0_fifo_extra;
1767
1768 fifo_state->plane[PLANE_CURSOR] = 63;
1769
1770 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1771
1772 /* spread the remainder evenly */
1773 for_each_plane_id_on_crtc(crtc, plane_id) {
1774 int plane_extra;
1775
1776 if (fifo_left == 0)
1777 break;
1778
1779 if ((active_planes & BIT(plane_id)) == 0)
1780 continue;
1781
1782 plane_extra = min(fifo_extra, fifo_left);
1783 fifo_state->plane[plane_id] += plane_extra;
1784 fifo_left -= plane_extra;
1785 }
1786
1787 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1788
1789 /* give it all to the first plane if none are active */
1790 if (active_planes == 0) {
1791 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1792 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1793 }
1794
1795 return 0;
1796}
1797
1798/* mark all levels starting from 'level' as invalid */
1799static void vlv_invalidate_wms(struct intel_crtc *crtc,
1800 struct vlv_wm_state *wm_state, int level)
1801{
1802 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1803
1804 for (; level < intel_wm_num_levels(dev_priv); level++) {
1805 enum plane_id plane_id;
1806
1807 for_each_plane_id_on_crtc(crtc, plane_id)
1808 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1809
1810 wm_state->sr[level].cursor = USHRT_MAX;
1811 wm_state->sr[level].plane = USHRT_MAX;
1812 }
1813}
1814
1815static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1816{
1817 if (wm > fifo_size)
1818 return USHRT_MAX;
1819 else
1820 return fifo_size - wm;
1821}
1822
1823/*
1824 * Starting from 'level' set all higher
1825 * levels to 'value' in the "raw" watermarks.
1826 */
1827static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1828 int level, enum plane_id plane_id, u16 value)
1829{
1830 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1831 int num_levels = intel_wm_num_levels(dev_priv);
1832 bool dirty = false;
1833
1834 for (; level < num_levels; level++) {
1835 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1836
1837 dirty |= raw->plane[plane_id] != value;
1838 raw->plane[plane_id] = value;
1839 }
1840
1841 return dirty;
1842}
1843
1844static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1845 const struct intel_plane_state *plane_state)
1846{
1847 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1848 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1849 enum plane_id plane_id = plane->id;
1850 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1851 int level;
1852 bool dirty = false;
1853
1854 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1855 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1856 goto out;
1857 }
1858
1859 for (level = 0; level < num_levels; level++) {
1860 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1861 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1862 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1863
1864 if (wm > max_wm)
1865 break;
1866
1867 dirty |= raw->plane[plane_id] != wm;
1868 raw->plane[plane_id] = wm;
1869 }
1870
1871 /* mark all higher levels as invalid */
1872 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1873
1874out:
1875 if (dirty)
1876 drm_dbg_kms(&dev_priv->drm,
1877 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1878 plane->base.name,
1879 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1880 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1881 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1882
1883 return dirty;
1884}
1885
1886static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1887 enum plane_id plane_id, int level)
1888{
1889 const struct g4x_pipe_wm *raw =
1890 &crtc_state->wm.vlv.raw[level];
1891 const struct vlv_fifo_state *fifo_state =
1892 &crtc_state->wm.vlv.fifo_state;
1893
1894 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1895}
1896
1897static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1898{
1899 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1900 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1901 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1902 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1903}
1904
1905static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1906{
1907 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1908 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1909 struct intel_atomic_state *state =
1910 to_intel_atomic_state(crtc_state->uapi.state);
1911 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1912 const struct vlv_fifo_state *fifo_state =
1913 &crtc_state->wm.vlv.fifo_state;
1914 int num_active_planes = hweight8(crtc_state->active_planes &
1915 ~BIT(PLANE_CURSOR));
1916 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1917 const struct intel_plane_state *old_plane_state;
1918 const struct intel_plane_state *new_plane_state;
1919 struct intel_plane *plane;
1920 enum plane_id plane_id;
1921 int level, ret, i;
1922 unsigned int dirty = 0;
1923
1924 for_each_oldnew_intel_plane_in_state(state, plane,
1925 old_plane_state,
1926 new_plane_state, i) {
1927 if (new_plane_state->hw.crtc != &crtc->base &&
1928 old_plane_state->hw.crtc != &crtc->base)
1929 continue;
1930
1931 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1932 dirty |= BIT(plane->id);
1933 }
1934
1935 /*
1936 * DSPARB registers may have been reset due to the
1937 * power well being turned off. Make sure we restore
1938 * them to a consistent state even if no primary/sprite
1939 * planes are initially active.
1940 */
1941 if (needs_modeset)
1942 crtc_state->fifo_changed = true;
1943
1944 if (!dirty)
1945 return 0;
1946
1947 /* cursor changes don't warrant a FIFO recompute */
1948 if (dirty & ~BIT(PLANE_CURSOR)) {
1949 const struct intel_crtc_state *old_crtc_state =
1950 intel_atomic_get_old_crtc_state(state, crtc);
1951 const struct vlv_fifo_state *old_fifo_state =
1952 &old_crtc_state->wm.vlv.fifo_state;
1953
1954 ret = vlv_compute_fifo(crtc_state);
1955 if (ret)
1956 return ret;
1957
1958 if (needs_modeset ||
1959 memcmp(old_fifo_state, fifo_state,
1960 sizeof(*fifo_state)) != 0)
1961 crtc_state->fifo_changed = true;
1962 }
1963
1964 /* initially allow all levels */
1965 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1966 /*
1967 * Note that enabling cxsr with no primary/sprite planes
1968 * enabled can wedge the pipe. Hence we only allow cxsr
1969 * with exactly one enabled primary/sprite plane.
1970 */
1971 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1972
1973 for (level = 0; level < wm_state->num_levels; level++) {
1974 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1975 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1976
1977 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1978 break;
1979
1980 for_each_plane_id_on_crtc(crtc, plane_id) {
1981 wm_state->wm[level].plane[plane_id] =
1982 vlv_invert_wm_value(raw->plane[plane_id],
1983 fifo_state->plane[plane_id]);
1984 }
1985
1986 wm_state->sr[level].plane =
1987 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1988 raw->plane[PLANE_SPRITE0],
1989 raw->plane[PLANE_SPRITE1]),
1990 sr_fifo_size);
1991
1992 wm_state->sr[level].cursor =
1993 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1994 63);
1995 }
1996
1997 if (level == 0)
1998 return -EINVAL;
1999
2000 /* limit to only levels we can actually handle */
2001 wm_state->num_levels = level;
2002
2003 /* invalidate the higher levels */
2004 vlv_invalidate_wms(crtc, wm_state, level);
2005
2006 return 0;
2007}
2008
2009#define VLV_FIFO(plane, value) \
2010 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2011
2012static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2013 struct intel_crtc *crtc)
2014{
2015 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2016 struct intel_uncore *uncore = &dev_priv->uncore;
2017 const struct intel_crtc_state *crtc_state =
2018 intel_atomic_get_new_crtc_state(state, crtc);
2019 const struct vlv_fifo_state *fifo_state =
2020 &crtc_state->wm.vlv.fifo_state;
2021 int sprite0_start, sprite1_start, fifo_size;
2022 u32 dsparb, dsparb2, dsparb3;
2023
2024 if (!crtc_state->fifo_changed)
2025 return;
2026
2027 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2028 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2029 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2030
2031 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2032 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2033
2034 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2035
2036 /*
2037 * uncore.lock serves a double purpose here. It allows us to
2038 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2039 * it protects the DSPARB registers from getting clobbered by
2040 * parallel updates from multiple pipes.
2041 *
2042 * intel_pipe_update_start() has already disabled interrupts
2043 * for us, so a plain spin_lock() is sufficient here.
2044 */
2045 spin_lock(&uncore->lock);
2046
2047 switch (crtc->pipe) {
2048 case PIPE_A:
2049 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2050 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2051
2052 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2053 VLV_FIFO(SPRITEB, 0xff));
2054 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2055 VLV_FIFO(SPRITEB, sprite1_start));
2056
2057 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2058 VLV_FIFO(SPRITEB_HI, 0x1));
2059 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2060 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2061
2062 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2063 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2064 break;
2065 case PIPE_B:
2066 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2067 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2068
2069 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2070 VLV_FIFO(SPRITED, 0xff));
2071 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2072 VLV_FIFO(SPRITED, sprite1_start));
2073
2074 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2075 VLV_FIFO(SPRITED_HI, 0xff));
2076 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2077 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2078
2079 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2080 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2081 break;
2082 case PIPE_C:
2083 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2084 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2085
2086 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2087 VLV_FIFO(SPRITEF, 0xff));
2088 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2089 VLV_FIFO(SPRITEF, sprite1_start));
2090
2091 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2092 VLV_FIFO(SPRITEF_HI, 0xff));
2093 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2094 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2095
2096 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2097 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2098 break;
2099 default:
2100 break;
2101 }
2102
2103 intel_uncore_posting_read_fw(uncore, DSPARB);
2104
2105 spin_unlock(&uncore->lock);
2106}
2107
2108#undef VLV_FIFO
2109
2110static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2111{
2112 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2113 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2114 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2115 struct intel_atomic_state *intel_state =
2116 to_intel_atomic_state(new_crtc_state->uapi.state);
2117 const struct intel_crtc_state *old_crtc_state =
2118 intel_atomic_get_old_crtc_state(intel_state, crtc);
2119 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2120 int level;
2121
2122 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2123 *intermediate = *optimal;
2124
2125 intermediate->cxsr = false;
2126 goto out;
2127 }
2128
2129 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2130 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2131 !new_crtc_state->disable_cxsr;
2132
2133 for (level = 0; level < intermediate->num_levels; level++) {
2134 enum plane_id plane_id;
2135
2136 for_each_plane_id_on_crtc(crtc, plane_id) {
2137 intermediate->wm[level].plane[plane_id] =
2138 min(optimal->wm[level].plane[plane_id],
2139 active->wm[level].plane[plane_id]);
2140 }
2141
2142 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2143 active->sr[level].plane);
2144 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2145 active->sr[level].cursor);
2146 }
2147
2148 vlv_invalidate_wms(crtc, intermediate, level);
2149
2150out:
2151 /*
2152 * If our intermediate WM are identical to the final WM, then we can
2153 * omit the post-vblank programming; only update if it's different.
2154 */
2155 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2156 new_crtc_state->wm.need_postvbl_update = true;
2157
2158 return 0;
2159}
2160
2161static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2162 struct vlv_wm_values *wm)
2163{
2164 struct intel_crtc *crtc;
2165 int num_active_pipes = 0;
2166
2167 wm->level = dev_priv->wm.max_level;
2168 wm->cxsr = true;
2169
2170 for_each_intel_crtc(&dev_priv->drm, crtc) {
2171 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2172
2173 if (!crtc->active)
2174 continue;
2175
2176 if (!wm_state->cxsr)
2177 wm->cxsr = false;
2178
2179 num_active_pipes++;
2180 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2181 }
2182
2183 if (num_active_pipes != 1)
2184 wm->cxsr = false;
2185
2186 if (num_active_pipes > 1)
2187 wm->level = VLV_WM_LEVEL_PM2;
2188
2189 for_each_intel_crtc(&dev_priv->drm, crtc) {
2190 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2191 enum pipe pipe = crtc->pipe;
2192
2193 wm->pipe[pipe] = wm_state->wm[wm->level];
2194 if (crtc->active && wm->cxsr)
2195 wm->sr = wm_state->sr[wm->level];
2196
2197 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2198 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2199 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2200 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2201 }
2202}
2203
2204static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2205{
2206 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2207 struct vlv_wm_values new_wm = {};
2208
2209 vlv_merge_wm(dev_priv, &new_wm);
2210
2211 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2212 return;
2213
2214 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2215 chv_set_memory_dvfs(dev_priv, false);
2216
2217 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2218 chv_set_memory_pm5(dev_priv, false);
2219
2220 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2221 _intel_set_memory_cxsr(dev_priv, false);
2222
2223 vlv_write_wm_values(dev_priv, &new_wm);
2224
2225 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2226 _intel_set_memory_cxsr(dev_priv, true);
2227
2228 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2229 chv_set_memory_pm5(dev_priv, true);
2230
2231 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2232 chv_set_memory_dvfs(dev_priv, true);
2233
2234 *old_wm = new_wm;
2235}
2236
2237static void vlv_initial_watermarks(struct intel_atomic_state *state,
2238 struct intel_crtc *crtc)
2239{
2240 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2241 const struct intel_crtc_state *crtc_state =
2242 intel_atomic_get_new_crtc_state(state, crtc);
2243
2244 mutex_lock(&dev_priv->wm.wm_mutex);
2245 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2246 vlv_program_watermarks(dev_priv);
2247 mutex_unlock(&dev_priv->wm.wm_mutex);
2248}
2249
2250static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2251 struct intel_crtc *crtc)
2252{
2253 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2254 const struct intel_crtc_state *crtc_state =
2255 intel_atomic_get_new_crtc_state(state, crtc);
2256
2257 if (!crtc_state->wm.need_postvbl_update)
2258 return;
2259
2260 mutex_lock(&dev_priv->wm.wm_mutex);
2261 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2262 vlv_program_watermarks(dev_priv);
2263 mutex_unlock(&dev_priv->wm.wm_mutex);
2264}
2265
2266static void i965_update_wm(struct intel_crtc *unused_crtc)
2267{
2268 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2269 struct intel_crtc *crtc;
2270 int srwm = 1;
2271 int cursor_sr = 16;
2272 bool cxsr_enabled;
2273
2274 /* Calc sr entries for one plane configs */
2275 crtc = single_enabled_crtc(dev_priv);
2276 if (crtc) {
2277 /* self-refresh has much higher latency */
2278 static const int sr_latency_ns = 12000;
2279 const struct drm_display_mode *pipe_mode =
2280 &crtc->config->hw.pipe_mode;
2281 const struct drm_framebuffer *fb =
2282 crtc->base.primary->state->fb;
2283 int clock = pipe_mode->crtc_clock;
2284 int htotal = pipe_mode->crtc_htotal;
2285 int hdisplay = crtc->config->pipe_src_w;
2286 int cpp = fb->format->cpp[0];
2287 int entries;
2288
2289 entries = intel_wm_method2(clock, htotal,
2290 hdisplay, cpp, sr_latency_ns / 100);
2291 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2292 srwm = I965_FIFO_SIZE - entries;
2293 if (srwm < 0)
2294 srwm = 1;
2295 srwm &= 0x1ff;
2296 drm_dbg_kms(&dev_priv->drm,
2297 "self-refresh entries: %d, wm: %d\n",
2298 entries, srwm);
2299
2300 entries = intel_wm_method2(clock, htotal,
2301 crtc->base.cursor->state->crtc_w, 4,
2302 sr_latency_ns / 100);
2303 entries = DIV_ROUND_UP(entries,
2304 i965_cursor_wm_info.cacheline_size) +
2305 i965_cursor_wm_info.guard_size;
2306
2307 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2308 if (cursor_sr > i965_cursor_wm_info.max_wm)
2309 cursor_sr = i965_cursor_wm_info.max_wm;
2310
2311 drm_dbg_kms(&dev_priv->drm,
2312 "self-refresh watermark: display plane %d "
2313 "cursor %d\n", srwm, cursor_sr);
2314
2315 cxsr_enabled = true;
2316 } else {
2317 cxsr_enabled = false;
2318 /* Turn off self refresh if both pipes are enabled */
2319 intel_set_memory_cxsr(dev_priv, false);
2320 }
2321
2322 drm_dbg_kms(&dev_priv->drm,
2323 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2324 srwm);
2325
2326 /* 965 has limitations... */
2327 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2328 FW_WM(8, CURSORB) |
2329 FW_WM(8, PLANEB) |
2330 FW_WM(8, PLANEA));
2331 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2332 FW_WM(8, PLANEC_OLD));
2333 /* update cursor SR watermark */
2334 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2335
2336 if (cxsr_enabled)
2337 intel_set_memory_cxsr(dev_priv, true);
2338}
2339
2340#undef FW_WM
2341
2342static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2343{
2344 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2345 const struct intel_watermark_params *wm_info;
2346 u32 fwater_lo;
2347 u32 fwater_hi;
2348 int cwm, srwm = 1;
2349 int fifo_size;
2350 int planea_wm, planeb_wm;
2351 struct intel_crtc *crtc, *enabled = NULL;
2352
2353 if (IS_I945GM(dev_priv))
2354 wm_info = &i945_wm_info;
2355 else if (DISPLAY_VER(dev_priv) != 2)
2356 wm_info = &i915_wm_info;
2357 else
2358 wm_info = &i830_a_wm_info;
2359
2360 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2361 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2362 if (intel_crtc_active(crtc)) {
2363 const struct drm_display_mode *pipe_mode =
2364 &crtc->config->hw.pipe_mode;
2365 const struct drm_framebuffer *fb =
2366 crtc->base.primary->state->fb;
2367 int cpp;
2368
2369 if (DISPLAY_VER(dev_priv) == 2)
2370 cpp = 4;
2371 else
2372 cpp = fb->format->cpp[0];
2373
2374 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2375 wm_info, fifo_size, cpp,
2376 pessimal_latency_ns);
2377 enabled = crtc;
2378 } else {
2379 planea_wm = fifo_size - wm_info->guard_size;
2380 if (planea_wm > (long)wm_info->max_wm)
2381 planea_wm = wm_info->max_wm;
2382 }
2383
2384 if (DISPLAY_VER(dev_priv) == 2)
2385 wm_info = &i830_bc_wm_info;
2386
2387 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2388 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2389 if (intel_crtc_active(crtc)) {
2390 const struct drm_display_mode *pipe_mode =
2391 &crtc->config->hw.pipe_mode;
2392 const struct drm_framebuffer *fb =
2393 crtc->base.primary->state->fb;
2394 int cpp;
2395
2396 if (DISPLAY_VER(dev_priv) == 2)
2397 cpp = 4;
2398 else
2399 cpp = fb->format->cpp[0];
2400
2401 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2402 wm_info, fifo_size, cpp,
2403 pessimal_latency_ns);
2404 if (enabled == NULL)
2405 enabled = crtc;
2406 else
2407 enabled = NULL;
2408 } else {
2409 planeb_wm = fifo_size - wm_info->guard_size;
2410 if (planeb_wm > (long)wm_info->max_wm)
2411 planeb_wm = wm_info->max_wm;
2412 }
2413
2414 drm_dbg_kms(&dev_priv->drm,
2415 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2416
2417 if (IS_I915GM(dev_priv) && enabled) {
2418 struct drm_i915_gem_object *obj;
2419
2420 obj = intel_fb_obj(enabled->base.primary->state->fb);
2421
2422 /* self-refresh seems busted with untiled */
2423 if (!i915_gem_object_is_tiled(obj))
2424 enabled = NULL;
2425 }
2426
2427 /*
2428 * Overlay gets an aggressive default since video jitter is bad.
2429 */
2430 cwm = 2;
2431
2432 /* Play safe and disable self-refresh before adjusting watermarks. */
2433 intel_set_memory_cxsr(dev_priv, false);
2434
2435 /* Calc sr entries for one plane configs */
2436 if (HAS_FW_BLC(dev_priv) && enabled) {
2437 /* self-refresh has much higher latency */
2438 static const int sr_latency_ns = 6000;
2439 const struct drm_display_mode *pipe_mode =
2440 &enabled->config->hw.pipe_mode;
2441 const struct drm_framebuffer *fb =
2442 enabled->base.primary->state->fb;
2443 int clock = pipe_mode->crtc_clock;
2444 int htotal = pipe_mode->crtc_htotal;
2445 int hdisplay = enabled->config->pipe_src_w;
2446 int cpp;
2447 int entries;
2448
2449 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2450 cpp = 4;
2451 else
2452 cpp = fb->format->cpp[0];
2453
2454 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2455 sr_latency_ns / 100);
2456 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2457 drm_dbg_kms(&dev_priv->drm,
2458 "self-refresh entries: %d\n", entries);
2459 srwm = wm_info->fifo_size - entries;
2460 if (srwm < 0)
2461 srwm = 1;
2462
2463 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2464 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2465 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2466 else
2467 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2468 }
2469
2470 drm_dbg_kms(&dev_priv->drm,
2471 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2472 planea_wm, planeb_wm, cwm, srwm);
2473
2474 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2475 fwater_hi = (cwm & 0x1f);
2476
2477 /* Set request length to 8 cachelines per fetch */
2478 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2479 fwater_hi = fwater_hi | (1 << 8);
2480
2481 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2483
2484 if (enabled)
2485 intel_set_memory_cxsr(dev_priv, true);
2486}
2487
2488static void i845_update_wm(struct intel_crtc *unused_crtc)
2489{
2490 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2491 struct intel_crtc *crtc;
2492 const struct drm_display_mode *pipe_mode;
2493 u32 fwater_lo;
2494 int planea_wm;
2495
2496 crtc = single_enabled_crtc(dev_priv);
2497 if (crtc == NULL)
2498 return;
2499
2500 pipe_mode = &crtc->config->hw.pipe_mode;
2501 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2502 &i845_wm_info,
2503 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2504 4, pessimal_latency_ns);
2505 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2506 fwater_lo |= (3<<8) | planea_wm;
2507
2508 drm_dbg_kms(&dev_priv->drm,
2509 "Setting FIFO watermarks - A: %d\n", planea_wm);
2510
2511 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2512}
2513
2514/* latency must be in 0.1us units. */
2515static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2516 unsigned int cpp,
2517 unsigned int latency)
2518{
2519 unsigned int ret;
2520
2521 ret = intel_wm_method1(pixel_rate, cpp, latency);
2522 ret = DIV_ROUND_UP(ret, 64) + 2;
2523
2524 return ret;
2525}
2526
2527/* latency must be in 0.1us units. */
2528static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2529 unsigned int htotal,
2530 unsigned int width,
2531 unsigned int cpp,
2532 unsigned int latency)
2533{
2534 unsigned int ret;
2535
2536 ret = intel_wm_method2(pixel_rate, htotal,
2537 width, cpp, latency);
2538 ret = DIV_ROUND_UP(ret, 64) + 2;
2539
2540 return ret;
2541}
2542
2543static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2544{
2545 /*
2546 * Neither of these should be possible since this function shouldn't be
2547 * called if the CRTC is off or the plane is invisible. But let's be
2548 * extra paranoid to avoid a potential divide-by-zero if we screw up
2549 * elsewhere in the driver.
2550 */
2551 if (WARN_ON(!cpp))
2552 return 0;
2553 if (WARN_ON(!horiz_pixels))
2554 return 0;
2555
2556 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2557}
2558
2559struct ilk_wm_maximums {
2560 u16 pri;
2561 u16 spr;
2562 u16 cur;
2563 u16 fbc;
2564};
2565
2566/*
2567 * For both WM_PIPE and WM_LP.
2568 * mem_value must be in 0.1us units.
2569 */
2570static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2571 const struct intel_plane_state *plane_state,
2572 u32 mem_value, bool is_lp)
2573{
2574 u32 method1, method2;
2575 int cpp;
2576
2577 if (mem_value == 0)
2578 return U32_MAX;
2579
2580 if (!intel_wm_plane_visible(crtc_state, plane_state))
2581 return 0;
2582
2583 cpp = plane_state->hw.fb->format->cpp[0];
2584
2585 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2586
2587 if (!is_lp)
2588 return method1;
2589
2590 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2591 crtc_state->hw.pipe_mode.crtc_htotal,
2592 drm_rect_width(&plane_state->uapi.dst),
2593 cpp, mem_value);
2594
2595 return min(method1, method2);
2596}
2597
2598/*
2599 * For both WM_PIPE and WM_LP.
2600 * mem_value must be in 0.1us units.
2601 */
2602static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2603 const struct intel_plane_state *plane_state,
2604 u32 mem_value)
2605{
2606 u32 method1, method2;
2607 int cpp;
2608
2609 if (mem_value == 0)
2610 return U32_MAX;
2611
2612 if (!intel_wm_plane_visible(crtc_state, plane_state))
2613 return 0;
2614
2615 cpp = plane_state->hw.fb->format->cpp[0];
2616
2617 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2618 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2619 crtc_state->hw.pipe_mode.crtc_htotal,
2620 drm_rect_width(&plane_state->uapi.dst),
2621 cpp, mem_value);
2622 return min(method1, method2);
2623}
2624
2625/*
2626 * For both WM_PIPE and WM_LP.
2627 * mem_value must be in 0.1us units.
2628 */
2629static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2630 const struct intel_plane_state *plane_state,
2631 u32 mem_value)
2632{
2633 int cpp;
2634
2635 if (mem_value == 0)
2636 return U32_MAX;
2637
2638 if (!intel_wm_plane_visible(crtc_state, plane_state))
2639 return 0;
2640
2641 cpp = plane_state->hw.fb->format->cpp[0];
2642
2643 return ilk_wm_method2(crtc_state->pixel_rate,
2644 crtc_state->hw.pipe_mode.crtc_htotal,
2645 drm_rect_width(&plane_state->uapi.dst),
2646 cpp, mem_value);
2647}
2648
2649/* Only for WM_LP. */
2650static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2651 const struct intel_plane_state *plane_state,
2652 u32 pri_val)
2653{
2654 int cpp;
2655
2656 if (!intel_wm_plane_visible(crtc_state, plane_state))
2657 return 0;
2658
2659 cpp = plane_state->hw.fb->format->cpp[0];
2660
2661 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2662 cpp);
2663}
2664
2665static unsigned int
2666ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2667{
2668 if (DISPLAY_VER(dev_priv) >= 8)
2669 return 3072;
2670 else if (DISPLAY_VER(dev_priv) >= 7)
2671 return 768;
2672 else
2673 return 512;
2674}
2675
2676static unsigned int
2677ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2678 int level, bool is_sprite)
2679{
2680 if (DISPLAY_VER(dev_priv) >= 8)
2681 /* BDW primary/sprite plane watermarks */
2682 return level == 0 ? 255 : 2047;
2683 else if (DISPLAY_VER(dev_priv) >= 7)
2684 /* IVB/HSW primary/sprite plane watermarks */
2685 return level == 0 ? 127 : 1023;
2686 else if (!is_sprite)
2687 /* ILK/SNB primary plane watermarks */
2688 return level == 0 ? 127 : 511;
2689 else
2690 /* ILK/SNB sprite plane watermarks */
2691 return level == 0 ? 63 : 255;
2692}
2693
2694static unsigned int
2695ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2696{
2697 if (DISPLAY_VER(dev_priv) >= 7)
2698 return level == 0 ? 63 : 255;
2699 else
2700 return level == 0 ? 31 : 63;
2701}
2702
2703static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2704{
2705 if (DISPLAY_VER(dev_priv) >= 8)
2706 return 31;
2707 else
2708 return 15;
2709}
2710
2711/* Calculate the maximum primary/sprite plane watermark */
2712static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2713 int level,
2714 const struct intel_wm_config *config,
2715 enum intel_ddb_partitioning ddb_partitioning,
2716 bool is_sprite)
2717{
2718 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2719
2720 /* if sprites aren't enabled, sprites get nothing */
2721 if (is_sprite && !config->sprites_enabled)
2722 return 0;
2723
2724 /* HSW allows LP1+ watermarks even with multiple pipes */
2725 if (level == 0 || config->num_pipes_active > 1) {
2726 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2727
2728 /*
2729 * For some reason the non self refresh
2730 * FIFO size is only half of the self
2731 * refresh FIFO size on ILK/SNB.
2732 */
2733 if (DISPLAY_VER(dev_priv) <= 6)
2734 fifo_size /= 2;
2735 }
2736
2737 if (config->sprites_enabled) {
2738 /* level 0 is always calculated with 1:1 split */
2739 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2740 if (is_sprite)
2741 fifo_size *= 5;
2742 fifo_size /= 6;
2743 } else {
2744 fifo_size /= 2;
2745 }
2746 }
2747
2748 /* clamp to max that the registers can hold */
2749 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2750}
2751
2752/* Calculate the maximum cursor plane watermark */
2753static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2754 int level,
2755 const struct intel_wm_config *config)
2756{
2757 /* HSW LP1+ watermarks w/ multiple pipes */
2758 if (level > 0 && config->num_pipes_active > 1)
2759 return 64;
2760
2761 /* otherwise just report max that registers can hold */
2762 return ilk_cursor_wm_reg_max(dev_priv, level);
2763}
2764
2765static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2766 int level,
2767 const struct intel_wm_config *config,
2768 enum intel_ddb_partitioning ddb_partitioning,
2769 struct ilk_wm_maximums *max)
2770{
2771 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2772 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2773 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2774 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2775}
2776
2777static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2778 int level,
2779 struct ilk_wm_maximums *max)
2780{
2781 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2782 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2783 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2784 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2785}
2786
2787static bool ilk_validate_wm_level(int level,
2788 const struct ilk_wm_maximums *max,
2789 struct intel_wm_level *result)
2790{
2791 bool ret;
2792
2793 /* already determined to be invalid? */
2794 if (!result->enable)
2795 return false;
2796
2797 result->enable = result->pri_val <= max->pri &&
2798 result->spr_val <= max->spr &&
2799 result->cur_val <= max->cur;
2800
2801 ret = result->enable;
2802
2803 /*
2804 * HACK until we can pre-compute everything,
2805 * and thus fail gracefully if LP0 watermarks
2806 * are exceeded...
2807 */
2808 if (level == 0 && !result->enable) {
2809 if (result->pri_val > max->pri)
2810 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2811 level, result->pri_val, max->pri);
2812 if (result->spr_val > max->spr)
2813 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2814 level, result->spr_val, max->spr);
2815 if (result->cur_val > max->cur)
2816 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2817 level, result->cur_val, max->cur);
2818
2819 result->pri_val = min_t(u32, result->pri_val, max->pri);
2820 result->spr_val = min_t(u32, result->spr_val, max->spr);
2821 result->cur_val = min_t(u32, result->cur_val, max->cur);
2822 result->enable = true;
2823 }
2824
2825 return ret;
2826}
2827
2828static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2829 const struct intel_crtc *crtc,
2830 int level,
2831 struct intel_crtc_state *crtc_state,
2832 const struct intel_plane_state *pristate,
2833 const struct intel_plane_state *sprstate,
2834 const struct intel_plane_state *curstate,
2835 struct intel_wm_level *result)
2836{
2837 u16 pri_latency = dev_priv->wm.pri_latency[level];
2838 u16 spr_latency = dev_priv->wm.spr_latency[level];
2839 u16 cur_latency = dev_priv->wm.cur_latency[level];
2840
2841 /* WM1+ latency values stored in 0.5us units */
2842 if (level > 0) {
2843 pri_latency *= 5;
2844 spr_latency *= 5;
2845 cur_latency *= 5;
2846 }
2847
2848 if (pristate) {
2849 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2850 pri_latency, level);
2851 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2852 }
2853
2854 if (sprstate)
2855 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2856
2857 if (curstate)
2858 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2859
2860 result->enable = true;
2861}
2862
2863static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2864 u16 wm[8])
2865{
2866 struct intel_uncore *uncore = &dev_priv->uncore;
2867
2868 if (DISPLAY_VER(dev_priv) >= 9) {
2869 u32 val;
2870 int ret, i;
2871 int level, max_level = ilk_wm_max_level(dev_priv);
2872
2873 /* read the first set of memory latencies[0:3] */
2874 val = 0; /* data0 to be programmed to 0 for first set */
2875 ret = sandybridge_pcode_read(dev_priv,
2876 GEN9_PCODE_READ_MEM_LATENCY,
2877 &val, NULL);
2878
2879 if (ret) {
2880 drm_err(&dev_priv->drm,
2881 "SKL Mailbox read error = %d\n", ret);
2882 return;
2883 }
2884
2885 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2886 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2887 GEN9_MEM_LATENCY_LEVEL_MASK;
2888 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2889 GEN9_MEM_LATENCY_LEVEL_MASK;
2890 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2891 GEN9_MEM_LATENCY_LEVEL_MASK;
2892
2893 /* read the second set of memory latencies[4:7] */
2894 val = 1; /* data0 to be programmed to 1 for second set */
2895 ret = sandybridge_pcode_read(dev_priv,
2896 GEN9_PCODE_READ_MEM_LATENCY,
2897 &val, NULL);
2898 if (ret) {
2899 drm_err(&dev_priv->drm,
2900 "SKL Mailbox read error = %d\n", ret);
2901 return;
2902 }
2903
2904 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2905 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2906 GEN9_MEM_LATENCY_LEVEL_MASK;
2907 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2908 GEN9_MEM_LATENCY_LEVEL_MASK;
2909 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2910 GEN9_MEM_LATENCY_LEVEL_MASK;
2911
2912 /*
2913 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2914 * need to be disabled. We make sure to sanitize the values out
2915 * of the punit to satisfy this requirement.
2916 */
2917 for (level = 1; level <= max_level; level++) {
2918 if (wm[level] == 0) {
2919 for (i = level + 1; i <= max_level; i++)
2920 wm[i] = 0;
2921 break;
2922 }
2923 }
2924
2925 /*
2926 * WaWmMemoryReadLatency:skl+,glk
2927 *
2928 * punit doesn't take into account the read latency so we need
2929 * to add 2us to the various latency levels we retrieve from the
2930 * punit when level 0 response data us 0us.
2931 */
2932 if (wm[0] == 0) {
2933 wm[0] += 2;
2934 for (level = 1; level <= max_level; level++) {
2935 if (wm[level] == 0)
2936 break;
2937 wm[level] += 2;
2938 }
2939 }
2940
2941 /*
2942 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2943 * If we could not get dimm info enable this WA to prevent from
2944 * any underrun. If not able to get Dimm info assume 16GB dimm
2945 * to avoid any underrun.
2946 */
2947 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2948 wm[0] += 1;
2949
2950 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2951 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2952
2953 wm[0] = (sskpd >> 56) & 0xFF;
2954 if (wm[0] == 0)
2955 wm[0] = sskpd & 0xF;
2956 wm[1] = (sskpd >> 4) & 0xFF;
2957 wm[2] = (sskpd >> 12) & 0xFF;
2958 wm[3] = (sskpd >> 20) & 0x1FF;
2959 wm[4] = (sskpd >> 32) & 0x1FF;
2960 } else if (DISPLAY_VER(dev_priv) >= 6) {
2961 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2962
2963 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2964 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2965 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2966 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2967 } else if (DISPLAY_VER(dev_priv) >= 5) {
2968 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2969
2970 /* ILK primary LP0 latency is 700 ns */
2971 wm[0] = 7;
2972 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2973 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2974 } else {
2975 MISSING_CASE(INTEL_DEVID(dev_priv));
2976 }
2977}
2978
2979static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2980 u16 wm[5])
2981{
2982 /* ILK sprite LP0 latency is 1300 ns */
2983 if (DISPLAY_VER(dev_priv) == 5)
2984 wm[0] = 13;
2985}
2986
2987static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2988 u16 wm[5])
2989{
2990 /* ILK cursor LP0 latency is 1300 ns */
2991 if (DISPLAY_VER(dev_priv) == 5)
2992 wm[0] = 13;
2993}
2994
2995int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2996{
2997 /* how many WM levels are we expecting */
2998 if (HAS_HW_SAGV_WM(dev_priv))
2999 return 5;
3000 else if (DISPLAY_VER(dev_priv) >= 9)
3001 return 7;
3002 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3003 return 4;
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3005 return 3;
3006 else
3007 return 2;
3008}
3009
3010static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3011 const char *name,
3012 const u16 wm[])
3013{
3014 int level, max_level = ilk_wm_max_level(dev_priv);
3015
3016 for (level = 0; level <= max_level; level++) {
3017 unsigned int latency = wm[level];
3018
3019 if (latency == 0) {
3020 drm_dbg_kms(&dev_priv->drm,
3021 "%s WM%d latency not provided\n",
3022 name, level);
3023 continue;
3024 }
3025
3026 /*
3027 * - latencies are in us on gen9.
3028 * - before then, WM1+ latency values are in 0.5us units
3029 */
3030 if (DISPLAY_VER(dev_priv) >= 9)
3031 latency *= 10;
3032 else if (level > 0)
3033 latency *= 5;
3034
3035 drm_dbg_kms(&dev_priv->drm,
3036 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3037 wm[level], latency / 10, latency % 10);
3038 }
3039}
3040
3041static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3042 u16 wm[5], u16 min)
3043{
3044 int level, max_level = ilk_wm_max_level(dev_priv);
3045
3046 if (wm[0] >= min)
3047 return false;
3048
3049 wm[0] = max(wm[0], min);
3050 for (level = 1; level <= max_level; level++)
3051 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3052
3053 return true;
3054}
3055
3056static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3057{
3058 bool changed;
3059
3060 /*
3061 * The BIOS provided WM memory latency values are often
3062 * inadequate for high resolution displays. Adjust them.
3063 */
3064 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3065 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3066 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3067
3068 if (!changed)
3069 return;
3070
3071 drm_dbg_kms(&dev_priv->drm,
3072 "WM latency values increased to avoid potential underruns\n");
3073 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3074 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3075 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3076}
3077
3078static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3079{
3080 /*
3081 * On some SNB machines (Thinkpad X220 Tablet at least)
3082 * LP3 usage can cause vblank interrupts to be lost.
3083 * The DEIIR bit will go high but it looks like the CPU
3084 * never gets interrupted.
3085 *
3086 * It's not clear whether other interrupt source could
3087 * be affected or if this is somehow limited to vblank
3088 * interrupts only. To play it safe we disable LP3
3089 * watermarks entirely.
3090 */
3091 if (dev_priv->wm.pri_latency[3] == 0 &&
3092 dev_priv->wm.spr_latency[3] == 0 &&
3093 dev_priv->wm.cur_latency[3] == 0)
3094 return;
3095
3096 dev_priv->wm.pri_latency[3] = 0;
3097 dev_priv->wm.spr_latency[3] = 0;
3098 dev_priv->wm.cur_latency[3] = 0;
3099
3100 drm_dbg_kms(&dev_priv->drm,
3101 "LP3 watermarks disabled due to potential for lost interrupts\n");
3102 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3103 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3104 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3105}
3106
3107static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3108{
3109 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3110
3111 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3112 sizeof(dev_priv->wm.pri_latency));
3113 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3114 sizeof(dev_priv->wm.pri_latency));
3115
3116 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3117 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3118
3119 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3120 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3121 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3122
3123 if (DISPLAY_VER(dev_priv) == 6) {
3124 snb_wm_latency_quirk(dev_priv);
3125 snb_wm_lp3_irq_quirk(dev_priv);
3126 }
3127}
3128
3129static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3130{
3131 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3132 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3133}
3134
3135static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3136 struct intel_pipe_wm *pipe_wm)
3137{
3138 /* LP0 watermark maximums depend on this pipe alone */
3139 const struct intel_wm_config config = {
3140 .num_pipes_active = 1,
3141 .sprites_enabled = pipe_wm->sprites_enabled,
3142 .sprites_scaled = pipe_wm->sprites_scaled,
3143 };
3144 struct ilk_wm_maximums max;
3145
3146 /* LP0 watermarks always use 1/2 DDB partitioning */
3147 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3148
3149 /* At least LP0 must be valid */
3150 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3151 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3152 return false;
3153 }
3154
3155 return true;
3156}
3157
3158/* Compute new watermarks for the pipe */
3159static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3160{
3161 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3162 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3163 struct intel_pipe_wm *pipe_wm;
3164 struct intel_plane *plane;
3165 const struct intel_plane_state *plane_state;
3166 const struct intel_plane_state *pristate = NULL;
3167 const struct intel_plane_state *sprstate = NULL;
3168 const struct intel_plane_state *curstate = NULL;
3169 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3170 struct ilk_wm_maximums max;
3171
3172 pipe_wm = &crtc_state->wm.ilk.optimal;
3173
3174 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3175 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3176 pristate = plane_state;
3177 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3178 sprstate = plane_state;
3179 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3180 curstate = plane_state;
3181 }
3182
3183 pipe_wm->pipe_enabled = crtc_state->hw.active;
3184 if (sprstate) {
3185 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3186 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3187 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3188 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3189 }
3190
3191 usable_level = max_level;
3192
3193 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3194 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3195 usable_level = 1;
3196
3197 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3198 if (pipe_wm->sprites_scaled)
3199 usable_level = 0;
3200
3201 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3202 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3203 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3204
3205 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3206 return -EINVAL;
3207
3208 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3209
3210 for (level = 1; level <= usable_level; level++) {
3211 struct intel_wm_level *wm = &pipe_wm->wm[level];
3212
3213 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3214 pristate, sprstate, curstate, wm);
3215
3216 /*
3217 * Disable any watermark level that exceeds the
3218 * register maximums since such watermarks are
3219 * always invalid.
3220 */
3221 if (!ilk_validate_wm_level(level, &max, wm)) {
3222 memset(wm, 0, sizeof(*wm));
3223 break;
3224 }
3225 }
3226
3227 return 0;
3228}
3229
3230/*
3231 * Build a set of 'intermediate' watermark values that satisfy both the old
3232 * state and the new state. These can be programmed to the hardware
3233 * immediately.
3234 */
3235static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3236{
3237 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3238 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3239 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3240 struct intel_atomic_state *intel_state =
3241 to_intel_atomic_state(newstate->uapi.state);
3242 const struct intel_crtc_state *oldstate =
3243 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3244 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3245 int level, max_level = ilk_wm_max_level(dev_priv);
3246
3247 /*
3248 * Start with the final, target watermarks, then combine with the
3249 * currently active watermarks to get values that are safe both before
3250 * and after the vblank.
3251 */
3252 *a = newstate->wm.ilk.optimal;
3253 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3254 intel_state->skip_intermediate_wm)
3255 return 0;
3256
3257 a->pipe_enabled |= b->pipe_enabled;
3258 a->sprites_enabled |= b->sprites_enabled;
3259 a->sprites_scaled |= b->sprites_scaled;
3260
3261 for (level = 0; level <= max_level; level++) {
3262 struct intel_wm_level *a_wm = &a->wm[level];
3263 const struct intel_wm_level *b_wm = &b->wm[level];
3264
3265 a_wm->enable &= b_wm->enable;
3266 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3267 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3268 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3269 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3270 }
3271
3272 /*
3273 * We need to make sure that these merged watermark values are
3274 * actually a valid configuration themselves. If they're not,
3275 * there's no safe way to transition from the old state to
3276 * the new state, so we need to fail the atomic transaction.
3277 */
3278 if (!ilk_validate_pipe_wm(dev_priv, a))
3279 return -EINVAL;
3280
3281 /*
3282 * If our intermediate WM are identical to the final WM, then we can
3283 * omit the post-vblank programming; only update if it's different.
3284 */
3285 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3286 newstate->wm.need_postvbl_update = true;
3287
3288 return 0;
3289}
3290
3291/*
3292 * Merge the watermarks from all active pipes for a specific level.
3293 */
3294static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3295 int level,
3296 struct intel_wm_level *ret_wm)
3297{
3298 const struct intel_crtc *intel_crtc;
3299
3300 ret_wm->enable = true;
3301
3302 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3303 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3304 const struct intel_wm_level *wm = &active->wm[level];
3305
3306 if (!active->pipe_enabled)
3307 continue;
3308
3309 /*
3310 * The watermark values may have been used in the past,
3311 * so we must maintain them in the registers for some
3312 * time even if the level is now disabled.
3313 */
3314 if (!wm->enable)
3315 ret_wm->enable = false;
3316
3317 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3318 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3319 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3320 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3321 }
3322}
3323
3324/*
3325 * Merge all low power watermarks for all active pipes.
3326 */
3327static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3328 const struct intel_wm_config *config,
3329 const struct ilk_wm_maximums *max,
3330 struct intel_pipe_wm *merged)
3331{
3332 int level, max_level = ilk_wm_max_level(dev_priv);
3333 int last_enabled_level = max_level;
3334
3335 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3336 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3337 config->num_pipes_active > 1)
3338 last_enabled_level = 0;
3339
3340 /* ILK: FBC WM must be disabled always */
3341 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3342
3343 /* merge each WM1+ level */
3344 for (level = 1; level <= max_level; level++) {
3345 struct intel_wm_level *wm = &merged->wm[level];
3346
3347 ilk_merge_wm_level(dev_priv, level, wm);
3348
3349 if (level > last_enabled_level)
3350 wm->enable = false;
3351 else if (!ilk_validate_wm_level(level, max, wm))
3352 /* make sure all following levels get disabled */
3353 last_enabled_level = level - 1;
3354
3355 /*
3356 * The spec says it is preferred to disable
3357 * FBC WMs instead of disabling a WM level.
3358 */
3359 if (wm->fbc_val > max->fbc) {
3360 if (wm->enable)
3361 merged->fbc_wm_enabled = false;
3362 wm->fbc_val = 0;
3363 }
3364 }
3365
3366 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3367 /*
3368 * FIXME this is racy. FBC might get enabled later.
3369 * What we should check here is whether FBC can be
3370 * enabled sometime later.
3371 */
3372 if (DISPLAY_VER(dev_priv) == 5 && !merged->fbc_wm_enabled &&
3373 intel_fbc_is_active(dev_priv)) {
3374 for (level = 2; level <= max_level; level++) {
3375 struct intel_wm_level *wm = &merged->wm[level];
3376
3377 wm->enable = false;
3378 }
3379 }
3380}
3381
3382static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3383{
3384 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3385 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3386}
3387
3388/* The value we need to program into the WM_LPx latency field */
3389static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3390 int level)
3391{
3392 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3393 return 2 * level;
3394 else
3395 return dev_priv->wm.pri_latency[level];
3396}
3397
3398static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3399 const struct intel_pipe_wm *merged,
3400 enum intel_ddb_partitioning partitioning,
3401 struct ilk_wm_values *results)
3402{
3403 struct intel_crtc *intel_crtc;
3404 int level, wm_lp;
3405
3406 results->enable_fbc_wm = merged->fbc_wm_enabled;
3407 results->partitioning = partitioning;
3408
3409 /* LP1+ register values */
3410 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3411 const struct intel_wm_level *r;
3412
3413 level = ilk_wm_lp_to_level(wm_lp, merged);
3414
3415 r = &merged->wm[level];
3416
3417 /*
3418 * Maintain the watermark values even if the level is
3419 * disabled. Doing otherwise could cause underruns.
3420 */
3421 results->wm_lp[wm_lp - 1] =
3422 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3423 (r->pri_val << WM1_LP_SR_SHIFT) |
3424 r->cur_val;
3425
3426 if (r->enable)
3427 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3428
3429 if (DISPLAY_VER(dev_priv) >= 8)
3430 results->wm_lp[wm_lp - 1] |=
3431 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3432 else
3433 results->wm_lp[wm_lp - 1] |=
3434 r->fbc_val << WM1_LP_FBC_SHIFT;
3435
3436 /*
3437 * Always set WM1S_LP_EN when spr_val != 0, even if the
3438 * level is disabled. Doing otherwise could cause underruns.
3439 */
3440 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3441 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3442 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3443 } else
3444 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3445 }
3446
3447 /* LP0 register values */
3448 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3449 enum pipe pipe = intel_crtc->pipe;
3450 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3451 const struct intel_wm_level *r = &pipe_wm->wm[0];
3452
3453 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3454 continue;
3455
3456 results->wm_pipe[pipe] =
3457 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3458 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3459 r->cur_val;
3460 }
3461}
3462
3463/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3464 * case both are at the same level. Prefer r1 in case they're the same. */
3465static struct intel_pipe_wm *
3466ilk_find_best_result(struct drm_i915_private *dev_priv,
3467 struct intel_pipe_wm *r1,
3468 struct intel_pipe_wm *r2)
3469{
3470 int level, max_level = ilk_wm_max_level(dev_priv);
3471 int level1 = 0, level2 = 0;
3472
3473 for (level = 1; level <= max_level; level++) {
3474 if (r1->wm[level].enable)
3475 level1 = level;
3476 if (r2->wm[level].enable)
3477 level2 = level;
3478 }
3479
3480 if (level1 == level2) {
3481 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3482 return r2;
3483 else
3484 return r1;
3485 } else if (level1 > level2) {
3486 return r1;
3487 } else {
3488 return r2;
3489 }
3490}
3491
3492/* dirty bits used to track which watermarks need changes */
3493#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3494#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3495#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3496#define WM_DIRTY_FBC (1 << 24)
3497#define WM_DIRTY_DDB (1 << 25)
3498
3499static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3500 const struct ilk_wm_values *old,
3501 const struct ilk_wm_values *new)
3502{
3503 unsigned int dirty = 0;
3504 enum pipe pipe;
3505 int wm_lp;
3506
3507 for_each_pipe(dev_priv, pipe) {
3508 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3509 dirty |= WM_DIRTY_PIPE(pipe);
3510 /* Must disable LP1+ watermarks too */
3511 dirty |= WM_DIRTY_LP_ALL;
3512 }
3513 }
3514
3515 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3516 dirty |= WM_DIRTY_FBC;
3517 /* Must disable LP1+ watermarks too */
3518 dirty |= WM_DIRTY_LP_ALL;
3519 }
3520
3521 if (old->partitioning != new->partitioning) {
3522 dirty |= WM_DIRTY_DDB;
3523 /* Must disable LP1+ watermarks too */
3524 dirty |= WM_DIRTY_LP_ALL;
3525 }
3526
3527 /* LP1+ watermarks already deemed dirty, no need to continue */
3528 if (dirty & WM_DIRTY_LP_ALL)
3529 return dirty;
3530
3531 /* Find the lowest numbered LP1+ watermark in need of an update... */
3532 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3533 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3534 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3535 break;
3536 }
3537
3538 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3539 for (; wm_lp <= 3; wm_lp++)
3540 dirty |= WM_DIRTY_LP(wm_lp);
3541
3542 return dirty;
3543}
3544
3545static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3546 unsigned int dirty)
3547{
3548 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3549 bool changed = false;
3550
3551 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3552 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3553 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3554 changed = true;
3555 }
3556 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3557 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3558 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3559 changed = true;
3560 }
3561 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3562 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3563 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3564 changed = true;
3565 }
3566
3567 /*
3568 * Don't touch WM1S_LP_EN here.
3569 * Doing so could cause underruns.
3570 */
3571
3572 return changed;
3573}
3574
3575/*
3576 * The spec says we shouldn't write when we don't need, because every write
3577 * causes WMs to be re-evaluated, expending some power.
3578 */
3579static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3580 struct ilk_wm_values *results)
3581{
3582 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3583 unsigned int dirty;
3584 u32 val;
3585
3586 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3587 if (!dirty)
3588 return;
3589
3590 _ilk_disable_lp_wm(dev_priv, dirty);
3591
3592 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3593 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3594 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3595 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3596 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3597 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3598
3599 if (dirty & WM_DIRTY_DDB) {
3600 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3601 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3602 if (results->partitioning == INTEL_DDB_PART_1_2)
3603 val &= ~WM_MISC_DATA_PARTITION_5_6;
3604 else
3605 val |= WM_MISC_DATA_PARTITION_5_6;
3606 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3607 } else {
3608 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3609 if (results->partitioning == INTEL_DDB_PART_1_2)
3610 val &= ~DISP_DATA_PARTITION_5_6;
3611 else
3612 val |= DISP_DATA_PARTITION_5_6;
3613 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3614 }
3615 }
3616
3617 if (dirty & WM_DIRTY_FBC) {
3618 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3619 if (results->enable_fbc_wm)
3620 val &= ~DISP_FBC_WM_DIS;
3621 else
3622 val |= DISP_FBC_WM_DIS;
3623 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3624 }
3625
3626 if (dirty & WM_DIRTY_LP(1) &&
3627 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3628 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3629
3630 if (DISPLAY_VER(dev_priv) >= 7) {
3631 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3632 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3633 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3634 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3635 }
3636
3637 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3638 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3639 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3640 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3641 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3642 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3643
3644 dev_priv->wm.hw = *results;
3645}
3646
3647bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3648{
3649 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3650}
3651
3652u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3653{
3654 u8 enabled_slices = 0;
3655 enum dbuf_slice slice;
3656
3657 for_each_dbuf_slice(dev_priv, slice) {
3658 if (intel_uncore_read(&dev_priv->uncore,
3659 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3660 enabled_slices |= BIT(slice);
3661 }
3662
3663 return enabled_slices;
3664}
3665
3666/*
3667 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3668 * so assume we'll always need it in order to avoid underruns.
3669 */
3670static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3671{
3672 return DISPLAY_VER(dev_priv) == 9;
3673}
3674
3675static bool
3676intel_has_sagv(struct drm_i915_private *dev_priv)
3677{
3678 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3679 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3680}
3681
3682static void
3683skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3684{
3685 if (DISPLAY_VER(dev_priv) >= 12) {
3686 u32 val = 0;
3687 int ret;
3688
3689 ret = sandybridge_pcode_read(dev_priv,
3690 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3691 &val, NULL);
3692 if (!ret) {
3693 dev_priv->sagv_block_time_us = val;
3694 return;
3695 }
3696
3697 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3698 } else if (DISPLAY_VER(dev_priv) == 11) {
3699 dev_priv->sagv_block_time_us = 10;
3700 return;
3701 } else if (DISPLAY_VER(dev_priv) == 10) {
3702 dev_priv->sagv_block_time_us = 20;
3703 return;
3704 } else if (DISPLAY_VER(dev_priv) == 9) {
3705 dev_priv->sagv_block_time_us = 30;
3706 return;
3707 } else {
3708 MISSING_CASE(DISPLAY_VER(dev_priv));
3709 }
3710
3711 /* Default to an unusable block time */
3712 dev_priv->sagv_block_time_us = -1;
3713}
3714
3715/*
3716 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3717 * depending on power and performance requirements. The display engine access
3718 * to system memory is blocked during the adjustment time. Because of the
3719 * blocking time, having this enabled can cause full system hangs and/or pipe
3720 * underruns if we don't meet all of the following requirements:
3721 *
3722 * - <= 1 pipe enabled
3723 * - All planes can enable watermarks for latencies >= SAGV engine block time
3724 * - We're not using an interlaced display configuration
3725 */
3726static int
3727intel_enable_sagv(struct drm_i915_private *dev_priv)
3728{
3729 int ret;
3730
3731 if (!intel_has_sagv(dev_priv))
3732 return 0;
3733
3734 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3735 return 0;
3736
3737 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3738 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3739 GEN9_SAGV_ENABLE);
3740
3741 /* We don't need to wait for SAGV when enabling */
3742
3743 /*
3744 * Some skl systems, pre-release machines in particular,
3745 * don't actually have SAGV.
3746 */
3747 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3748 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3749 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3750 return 0;
3751 } else if (ret < 0) {
3752 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3753 return ret;
3754 }
3755
3756 dev_priv->sagv_status = I915_SAGV_ENABLED;
3757 return 0;
3758}
3759
3760static int
3761intel_disable_sagv(struct drm_i915_private *dev_priv)
3762{
3763 int ret;
3764
3765 if (!intel_has_sagv(dev_priv))
3766 return 0;
3767
3768 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3769 return 0;
3770
3771 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3772 /* bspec says to keep retrying for at least 1 ms */
3773 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3774 GEN9_SAGV_DISABLE,
3775 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3776 1);
3777 /*
3778 * Some skl systems, pre-release machines in particular,
3779 * don't actually have SAGV.
3780 */
3781 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3782 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3783 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3784 return 0;
3785 } else if (ret < 0) {
3786 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3787 return ret;
3788 }
3789
3790 dev_priv->sagv_status = I915_SAGV_DISABLED;
3791 return 0;
3792}
3793
3794void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3795{
3796 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3797 const struct intel_bw_state *new_bw_state;
3798 const struct intel_bw_state *old_bw_state;
3799 u32 new_mask = 0;
3800
3801 /*
3802 * Just return if we can't control SAGV or don't have it.
3803 * This is different from situation when we have SAGV but just can't
3804 * afford it due to DBuf limitation - in case if SAGV is completely
3805 * disabled in a BIOS, we are not even allowed to send a PCode request,
3806 * as it will throw an error. So have to check it here.
3807 */
3808 if (!intel_has_sagv(dev_priv))
3809 return;
3810
3811 new_bw_state = intel_atomic_get_new_bw_state(state);
3812 if (!new_bw_state)
3813 return;
3814
3815 if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3816 intel_disable_sagv(dev_priv);
3817 return;
3818 }
3819
3820 old_bw_state = intel_atomic_get_old_bw_state(state);
3821 /*
3822 * Nothing to mask
3823 */
3824 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3825 return;
3826
3827 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3828
3829 /*
3830 * If new mask is zero - means there is nothing to mask,
3831 * we can only unmask, which should be done in unmask.
3832 */
3833 if (!new_mask)
3834 return;
3835
3836 /*
3837 * Restrict required qgv points before updating the configuration.
3838 * According to BSpec we can't mask and unmask qgv points at the same
3839 * time. Also masking should be done before updating the configuration
3840 * and unmasking afterwards.
3841 */
3842 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3843}
3844
3845void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3846{
3847 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3848 const struct intel_bw_state *new_bw_state;
3849 const struct intel_bw_state *old_bw_state;
3850 u32 new_mask = 0;
3851
3852 /*
3853 * Just return if we can't control SAGV or don't have it.
3854 * This is different from situation when we have SAGV but just can't
3855 * afford it due to DBuf limitation - in case if SAGV is completely
3856 * disabled in a BIOS, we are not even allowed to send a PCode request,
3857 * as it will throw an error. So have to check it here.
3858 */
3859 if (!intel_has_sagv(dev_priv))
3860 return;
3861
3862 new_bw_state = intel_atomic_get_new_bw_state(state);
3863 if (!new_bw_state)
3864 return;
3865
3866 if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3867 intel_enable_sagv(dev_priv);
3868 return;
3869 }
3870
3871 old_bw_state = intel_atomic_get_old_bw_state(state);
3872 /*
3873 * Nothing to unmask
3874 */
3875 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3876 return;
3877
3878 new_mask = new_bw_state->qgv_points_mask;
3879
3880 /*
3881 * Allow required qgv points after updating the configuration.
3882 * According to BSpec we can't mask and unmask qgv points at the same
3883 * time. Also masking should be done before updating the configuration
3884 * and unmasking afterwards.
3885 */
3886 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3887}
3888
3889static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3890{
3891 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3892 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3893 enum plane_id plane_id;
3894 int max_level = INT_MAX;
3895
3896 if (!intel_has_sagv(dev_priv))
3897 return false;
3898
3899 if (!crtc_state->hw.active)
3900 return true;
3901
3902 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3903 return false;
3904
3905 for_each_plane_id_on_crtc(crtc, plane_id) {
3906 const struct skl_plane_wm *wm =
3907 &crtc_state->wm.skl.optimal.planes[plane_id];
3908 int level;
3909
3910 /* Skip this plane if it's not enabled */
3911 if (!wm->wm[0].enable)
3912 continue;
3913
3914 /* Find the highest enabled wm level for this plane */
3915 for (level = ilk_wm_max_level(dev_priv);
3916 !wm->wm[level].enable; --level)
3917 { }
3918
3919 /* Highest common enabled wm level for all planes */
3920 max_level = min(level, max_level);
3921 }
3922
3923 /* No enabled planes? */
3924 if (max_level == INT_MAX)
3925 return true;
3926
3927 for_each_plane_id_on_crtc(crtc, plane_id) {
3928 const struct skl_plane_wm *wm =
3929 &crtc_state->wm.skl.optimal.planes[plane_id];
3930
3931 /*
3932 * All enabled planes must have enabled a common wm level that
3933 * can tolerate memory latencies higher than sagv_block_time_us
3934 */
3935 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3936 return false;
3937 }
3938
3939 return true;
3940}
3941
3942static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3943{
3944 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3945 enum plane_id plane_id;
3946
3947 if (!crtc_state->hw.active)
3948 return true;
3949
3950 for_each_plane_id_on_crtc(crtc, plane_id) {
3951 const struct skl_plane_wm *wm =
3952 &crtc_state->wm.skl.optimal.planes[plane_id];
3953
3954 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3955 return false;
3956 }
3957
3958 return true;
3959}
3960
3961static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3962{
3963 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3964 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3965
3966 if (DISPLAY_VER(dev_priv) >= 12)
3967 return tgl_crtc_can_enable_sagv(crtc_state);
3968 else
3969 return skl_crtc_can_enable_sagv(crtc_state);
3970}
3971
3972bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3973 const struct intel_bw_state *bw_state)
3974{
3975 if (DISPLAY_VER(dev_priv) < 11 &&
3976 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3977 return false;
3978
3979 return bw_state->pipe_sagv_reject == 0;
3980}
3981
3982static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3983{
3984 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3985 int ret;
3986 struct intel_crtc *crtc;
3987 struct intel_crtc_state *new_crtc_state;
3988 struct intel_bw_state *new_bw_state = NULL;
3989 const struct intel_bw_state *old_bw_state = NULL;
3990 int i;
3991
3992 for_each_new_intel_crtc_in_state(state, crtc,
3993 new_crtc_state, i) {
3994 new_bw_state = intel_atomic_get_bw_state(state);
3995 if (IS_ERR(new_bw_state))
3996 return PTR_ERR(new_bw_state);
3997
3998 old_bw_state = intel_atomic_get_old_bw_state(state);
3999
4000 if (intel_crtc_can_enable_sagv(new_crtc_state))
4001 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4002 else
4003 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4004 }
4005
4006 if (!new_bw_state)
4007 return 0;
4008
4009 new_bw_state->active_pipes =
4010 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4011
4012 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4013 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4014 if (ret)
4015 return ret;
4016 }
4017
4018 for_each_new_intel_crtc_in_state(state, crtc,
4019 new_crtc_state, i) {
4020 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4021
4022 /*
4023 * We store use_sagv_wm in the crtc state rather than relying on
4024 * that bw state since we have no convenient way to get at the
4025 * latter from the plane commit hooks (especially in the legacy
4026 * cursor case)
4027 */
4028 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4029 DISPLAY_VER(dev_priv) >= 12 &&
4030 intel_can_enable_sagv(dev_priv, new_bw_state);
4031 }
4032
4033 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4034 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4035 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4036 if (ret)
4037 return ret;
4038 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4039 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4040 if (ret)
4041 return ret;
4042 }
4043
4044 return 0;
4045}
4046
4047static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4048{
4049 return INTEL_INFO(dev_priv)->dbuf.size /
4050 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4051}
4052
4053static void
4054skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4055 struct skl_ddb_entry *ddb)
4056{
4057 int slice_size = intel_dbuf_slice_size(dev_priv);
4058
4059 if (!slice_mask) {
4060 ddb->start = 0;
4061 ddb->end = 0;
4062 return;
4063 }
4064
4065 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4066 ddb->end = fls(slice_mask) * slice_size;
4067
4068 WARN_ON(ddb->start >= ddb->end);
4069 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4070}
4071
4072static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4073{
4074 struct skl_ddb_entry ddb;
4075
4076 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4077 slice_mask = BIT(DBUF_S1);
4078 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4079 slice_mask = BIT(DBUF_S3);
4080
4081 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4082
4083 return ddb.start;
4084}
4085
4086u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4087 const struct skl_ddb_entry *entry)
4088{
4089 int slice_size = intel_dbuf_slice_size(dev_priv);
4090 enum dbuf_slice start_slice, end_slice;
4091 u8 slice_mask = 0;
4092
4093 if (!skl_ddb_entry_size(entry))
4094 return 0;
4095
4096 start_slice = entry->start / slice_size;
4097 end_slice = (entry->end - 1) / slice_size;
4098
4099 /*
4100 * Per plane DDB entry can in a really worst case be on multiple slices
4101 * but single entry is anyway contigious.
4102 */
4103 while (start_slice <= end_slice) {
4104 slice_mask |= BIT(start_slice);
4105 start_slice++;
4106 }
4107
4108 return slice_mask;
4109}
4110
4111static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4112{
4113 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4114 int hdisplay, vdisplay;
4115
4116 if (!crtc_state->hw.active)
4117 return 0;
4118
4119 /*
4120 * Watermark/ddb requirement highly depends upon width of the
4121 * framebuffer, So instead of allocating DDB equally among pipes
4122 * distribute DDB based on resolution/width of the display.
4123 */
4124 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4125
4126 return hdisplay;
4127}
4128
4129static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4130 enum pipe for_pipe,
4131 unsigned int *weight_start,
4132 unsigned int *weight_end,
4133 unsigned int *weight_total)
4134{
4135 struct drm_i915_private *dev_priv =
4136 to_i915(dbuf_state->base.state->base.dev);
4137 enum pipe pipe;
4138
4139 *weight_start = 0;
4140 *weight_end = 0;
4141 *weight_total = 0;
4142
4143 for_each_pipe(dev_priv, pipe) {
4144 int weight = dbuf_state->weight[pipe];
4145
4146 /*
4147 * Do not account pipes using other slice sets
4148 * luckily as of current BSpec slice sets do not partially
4149 * intersect(pipes share either same one slice or same slice set
4150 * i.e no partial intersection), so it is enough to check for
4151 * equality for now.
4152 */
4153 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4154 continue;
4155
4156 *weight_total += weight;
4157 if (pipe < for_pipe) {
4158 *weight_start += weight;
4159 *weight_end += weight;
4160 } else if (pipe == for_pipe) {
4161 *weight_end += weight;
4162 }
4163 }
4164}
4165
4166static int
4167skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4168{
4169 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4170 unsigned int weight_total, weight_start, weight_end;
4171 const struct intel_dbuf_state *old_dbuf_state =
4172 intel_atomic_get_old_dbuf_state(state);
4173 struct intel_dbuf_state *new_dbuf_state =
4174 intel_atomic_get_new_dbuf_state(state);
4175 struct intel_crtc_state *crtc_state;
4176 struct skl_ddb_entry ddb_slices;
4177 enum pipe pipe = crtc->pipe;
4178 unsigned int mbus_offset = 0;
4179 u32 ddb_range_size;
4180 u32 dbuf_slice_mask;
4181 u32 start, end;
4182 int ret;
4183
4184 if (new_dbuf_state->weight[pipe] == 0) {
4185 new_dbuf_state->ddb[pipe].start = 0;
4186 new_dbuf_state->ddb[pipe].end = 0;
4187 goto out;
4188 }
4189
4190 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4191
4192 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4193 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4194 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4195
4196 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4197 &weight_start, &weight_end, &weight_total);
4198
4199 start = ddb_range_size * weight_start / weight_total;
4200 end = ddb_range_size * weight_end / weight_total;
4201
4202 new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start;
4203 new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end;
4204out:
4205 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4206 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4207 &new_dbuf_state->ddb[pipe]))
4208 return 0;
4209
4210 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4211 if (ret)
4212 return ret;
4213
4214 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4215 if (IS_ERR(crtc_state))
4216 return PTR_ERR(crtc_state);
4217
4218 /*
4219 * Used for checking overlaps, so we need absolute
4220 * offsets instead of MBUS relative offsets.
4221 */
4222 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4223 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4224
4225 drm_dbg_kms(&dev_priv->drm,
4226 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4227 crtc->base.base.id, crtc->base.name,
4228 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4229 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4230 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4231 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4232
4233 return 0;
4234}
4235
4236static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4237 int width, const struct drm_format_info *format,
4238 u64 modifier, unsigned int rotation,
4239 u32 plane_pixel_rate, struct skl_wm_params *wp,
4240 int color_plane);
4241static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4242 int level,
4243 unsigned int latency,
4244 const struct skl_wm_params *wp,
4245 const struct skl_wm_level *result_prev,
4246 struct skl_wm_level *result /* out */);
4247
4248static unsigned int
4249skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4250 int num_active)
4251{
4252 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4253 int level, max_level = ilk_wm_max_level(dev_priv);
4254 struct skl_wm_level wm = {};
4255 int ret, min_ddb_alloc = 0;
4256 struct skl_wm_params wp;
4257
4258 ret = skl_compute_wm_params(crtc_state, 256,
4259 drm_format_info(DRM_FORMAT_ARGB8888),
4260 DRM_FORMAT_MOD_LINEAR,
4261 DRM_MODE_ROTATE_0,
4262 crtc_state->pixel_rate, &wp, 0);
4263 drm_WARN_ON(&dev_priv->drm, ret);
4264
4265 for (level = 0; level <= max_level; level++) {
4266 unsigned int latency = dev_priv->wm.skl_latency[level];
4267
4268 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4269 if (wm.min_ddb_alloc == U16_MAX)
4270 break;
4271
4272 min_ddb_alloc = wm.min_ddb_alloc;
4273 }
4274
4275 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4276}
4277
4278static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4279 struct skl_ddb_entry *entry, u32 reg)
4280{
4281 entry->start = reg & DDB_ENTRY_MASK;
4282 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4283
4284 if (entry->end)
4285 entry->end += 1;
4286}
4287
4288static void
4289skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4290 const enum pipe pipe,
4291 const enum plane_id plane_id,
4292 struct skl_ddb_entry *ddb_y,
4293 struct skl_ddb_entry *ddb_uv)
4294{
4295 u32 val, val2;
4296 u32 fourcc = 0;
4297
4298 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4299 if (plane_id == PLANE_CURSOR) {
4300 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4301 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4302 return;
4303 }
4304
4305 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4306
4307 /* No DDB allocated for disabled planes */
4308 if (val & PLANE_CTL_ENABLE)
4309 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4310 val & PLANE_CTL_ORDER_RGBX,
4311 val & PLANE_CTL_ALPHA_MASK);
4312
4313 if (DISPLAY_VER(dev_priv) >= 11) {
4314 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4315 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4316 } else {
4317 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4318 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4319
4320 if (fourcc &&
4321 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4322 swap(val, val2);
4323
4324 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4325 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4326 }
4327}
4328
4329void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4330 struct skl_ddb_entry *ddb_y,
4331 struct skl_ddb_entry *ddb_uv)
4332{
4333 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4334 enum intel_display_power_domain power_domain;
4335 enum pipe pipe = crtc->pipe;
4336 intel_wakeref_t wakeref;
4337 enum plane_id plane_id;
4338
4339 power_domain = POWER_DOMAIN_PIPE(pipe);
4340 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4341 if (!wakeref)
4342 return;
4343
4344 for_each_plane_id_on_crtc(crtc, plane_id)
4345 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4346 plane_id,
4347 &ddb_y[plane_id],
4348 &ddb_uv[plane_id]);
4349
4350 intel_display_power_put(dev_priv, power_domain, wakeref);
4351}
4352
4353/*
4354 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4355 * The bspec defines downscale amount as:
4356 *
4357 * """
4358 * Horizontal down scale amount = maximum[1, Horizontal source size /
4359 * Horizontal destination size]
4360 * Vertical down scale amount = maximum[1, Vertical source size /
4361 * Vertical destination size]
4362 * Total down scale amount = Horizontal down scale amount *
4363 * Vertical down scale amount
4364 * """
4365 *
4366 * Return value is provided in 16.16 fixed point form to retain fractional part.
4367 * Caller should take care of dividing & rounding off the value.
4368 */
4369static uint_fixed_16_16_t
4370skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4371 const struct intel_plane_state *plane_state)
4372{
4373 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4374 u32 src_w, src_h, dst_w, dst_h;
4375 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4376 uint_fixed_16_16_t downscale_h, downscale_w;
4377
4378 if (drm_WARN_ON(&dev_priv->drm,
4379 !intel_wm_plane_visible(crtc_state, plane_state)))
4380 return u32_to_fixed16(0);
4381
4382 /*
4383 * Src coordinates are already rotated by 270 degrees for
4384 * the 90/270 degree plane rotation cases (to match the
4385 * GTT mapping), hence no need to account for rotation here.
4386 *
4387 * n.b., src is 16.16 fixed point, dst is whole integer.
4388 */
4389 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4390 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4391 dst_w = drm_rect_width(&plane_state->uapi.dst);
4392 dst_h = drm_rect_height(&plane_state->uapi.dst);
4393
4394 fp_w_ratio = div_fixed16(src_w, dst_w);
4395 fp_h_ratio = div_fixed16(src_h, dst_h);
4396 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4397 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4398
4399 return mul_fixed16(downscale_w, downscale_h);
4400}
4401
4402struct dbuf_slice_conf_entry {
4403 u8 active_pipes;
4404 u8 dbuf_mask[I915_MAX_PIPES];
4405 bool join_mbus;
4406};
4407
4408/*
4409 * Table taken from Bspec 12716
4410 * Pipes do have some preferred DBuf slice affinity,
4411 * plus there are some hardcoded requirements on how
4412 * those should be distributed for multipipe scenarios.
4413 * For more DBuf slices algorithm can get even more messy
4414 * and less readable, so decided to use a table almost
4415 * as is from BSpec itself - that way it is at least easier
4416 * to compare, change and check.
4417 */
4418static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4419/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4420{
4421 {
4422 .active_pipes = BIT(PIPE_A),
4423 .dbuf_mask = {
4424 [PIPE_A] = BIT(DBUF_S1),
4425 },
4426 },
4427 {
4428 .active_pipes = BIT(PIPE_B),
4429 .dbuf_mask = {
4430 [PIPE_B] = BIT(DBUF_S1),
4431 },
4432 },
4433 {
4434 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4435 .dbuf_mask = {
4436 [PIPE_A] = BIT(DBUF_S1),
4437 [PIPE_B] = BIT(DBUF_S2),
4438 },
4439 },
4440 {
4441 .active_pipes = BIT(PIPE_C),
4442 .dbuf_mask = {
4443 [PIPE_C] = BIT(DBUF_S2),
4444 },
4445 },
4446 {
4447 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4448 .dbuf_mask = {
4449 [PIPE_A] = BIT(DBUF_S1),
4450 [PIPE_C] = BIT(DBUF_S2),
4451 },
4452 },
4453 {
4454 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4455 .dbuf_mask = {
4456 [PIPE_B] = BIT(DBUF_S1),
4457 [PIPE_C] = BIT(DBUF_S2),
4458 },
4459 },
4460 {
4461 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4462 .dbuf_mask = {
4463 [PIPE_A] = BIT(DBUF_S1),
4464 [PIPE_B] = BIT(DBUF_S1),
4465 [PIPE_C] = BIT(DBUF_S2),
4466 },
4467 },
4468 {}
4469};
4470
4471/*
4472 * Table taken from Bspec 49255
4473 * Pipes do have some preferred DBuf slice affinity,
4474 * plus there are some hardcoded requirements on how
4475 * those should be distributed for multipipe scenarios.
4476 * For more DBuf slices algorithm can get even more messy
4477 * and less readable, so decided to use a table almost
4478 * as is from BSpec itself - that way it is at least easier
4479 * to compare, change and check.
4480 */
4481static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4482/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4483{
4484 {
4485 .active_pipes = BIT(PIPE_A),
4486 .dbuf_mask = {
4487 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4488 },
4489 },
4490 {
4491 .active_pipes = BIT(PIPE_B),
4492 .dbuf_mask = {
4493 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4494 },
4495 },
4496 {
4497 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4498 .dbuf_mask = {
4499 [PIPE_A] = BIT(DBUF_S2),
4500 [PIPE_B] = BIT(DBUF_S1),
4501 },
4502 },
4503 {
4504 .active_pipes = BIT(PIPE_C),
4505 .dbuf_mask = {
4506 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4507 },
4508 },
4509 {
4510 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4511 .dbuf_mask = {
4512 [PIPE_A] = BIT(DBUF_S1),
4513 [PIPE_C] = BIT(DBUF_S2),
4514 },
4515 },
4516 {
4517 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4518 .dbuf_mask = {
4519 [PIPE_B] = BIT(DBUF_S1),
4520 [PIPE_C] = BIT(DBUF_S2),
4521 },
4522 },
4523 {
4524 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4525 .dbuf_mask = {
4526 [PIPE_A] = BIT(DBUF_S1),
4527 [PIPE_B] = BIT(DBUF_S1),
4528 [PIPE_C] = BIT(DBUF_S2),
4529 },
4530 },
4531 {
4532 .active_pipes = BIT(PIPE_D),
4533 .dbuf_mask = {
4534 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4535 },
4536 },
4537 {
4538 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4539 .dbuf_mask = {
4540 [PIPE_A] = BIT(DBUF_S1),
4541 [PIPE_D] = BIT(DBUF_S2),
4542 },
4543 },
4544 {
4545 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4546 .dbuf_mask = {
4547 [PIPE_B] = BIT(DBUF_S1),
4548 [PIPE_D] = BIT(DBUF_S2),
4549 },
4550 },
4551 {
4552 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4553 .dbuf_mask = {
4554 [PIPE_A] = BIT(DBUF_S1),
4555 [PIPE_B] = BIT(DBUF_S1),
4556 [PIPE_D] = BIT(DBUF_S2),
4557 },
4558 },
4559 {
4560 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4561 .dbuf_mask = {
4562 [PIPE_C] = BIT(DBUF_S1),
4563 [PIPE_D] = BIT(DBUF_S2),
4564 },
4565 },
4566 {
4567 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4568 .dbuf_mask = {
4569 [PIPE_A] = BIT(DBUF_S1),
4570 [PIPE_C] = BIT(DBUF_S2),
4571 [PIPE_D] = BIT(DBUF_S2),
4572 },
4573 },
4574 {
4575 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4576 .dbuf_mask = {
4577 [PIPE_B] = BIT(DBUF_S1),
4578 [PIPE_C] = BIT(DBUF_S2),
4579 [PIPE_D] = BIT(DBUF_S2),
4580 },
4581 },
4582 {
4583 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4584 .dbuf_mask = {
4585 [PIPE_A] = BIT(DBUF_S1),
4586 [PIPE_B] = BIT(DBUF_S1),
4587 [PIPE_C] = BIT(DBUF_S2),
4588 [PIPE_D] = BIT(DBUF_S2),
4589 },
4590 },
4591 {}
4592};
4593
4594static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4595 {
4596 .active_pipes = BIT(PIPE_A),
4597 .dbuf_mask = {
4598 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4599 },
4600 .join_mbus = true,
4601 },
4602 {
4603 .active_pipes = BIT(PIPE_B),
4604 .dbuf_mask = {
4605 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4606 },
4607 .join_mbus = true,
4608 },
4609 {
4610 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4611 .dbuf_mask = {
4612 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4613 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4614 },
4615 },
4616 {
4617 .active_pipes = BIT(PIPE_C),
4618 .dbuf_mask = {
4619 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4620 },
4621 },
4622 {
4623 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4624 .dbuf_mask = {
4625 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4626 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4627 },
4628 },
4629 {
4630 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4631 .dbuf_mask = {
4632 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4633 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4634 },
4635 },
4636 {
4637 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4638 .dbuf_mask = {
4639 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4640 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4641 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4642 },
4643 },
4644 {
4645 .active_pipes = BIT(PIPE_D),
4646 .dbuf_mask = {
4647 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4648 },
4649 },
4650 {
4651 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4652 .dbuf_mask = {
4653 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4654 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4655 },
4656 },
4657 {
4658 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4659 .dbuf_mask = {
4660 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4661 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4662 },
4663 },
4664 {
4665 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4666 .dbuf_mask = {
4667 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4668 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4669 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4670 },
4671 },
4672 {
4673 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4674 .dbuf_mask = {
4675 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4676 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4677 },
4678 },
4679 {
4680 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4681 .dbuf_mask = {
4682 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4683 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4684 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4685 },
4686 },
4687 {
4688 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4689 .dbuf_mask = {
4690 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4691 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4692 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4693 },
4694 },
4695 {
4696 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4697 .dbuf_mask = {
4698 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4699 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4700 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4701 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4702 },
4703 },
4704 {}
4705
4706};
4707
4708static bool check_mbus_joined(u8 active_pipes,
4709 const struct dbuf_slice_conf_entry *dbuf_slices)
4710{
4711 int i;
4712
4713 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4714 if (dbuf_slices[i].active_pipes == active_pipes)
4715 return dbuf_slices[i].join_mbus;
4716 }
4717 return false;
4718}
4719
4720static bool adlp_check_mbus_joined(u8 active_pipes)
4721{
4722 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4723}
4724
4725static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4726 const struct dbuf_slice_conf_entry *dbuf_slices)
4727{
4728 int i;
4729
4730 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4731 if (dbuf_slices[i].active_pipes == active_pipes)
4732 return dbuf_slices[i].dbuf_mask[pipe];
4733 }
4734 return 0;
4735}
4736
4737/*
4738 * This function finds an entry with same enabled pipe configuration and
4739 * returns correspondent DBuf slice mask as stated in BSpec for particular
4740 * platform.
4741 */
4742static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4743{
4744 /*
4745 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4746 * required calculating "pipe ratio" in order to determine
4747 * if one or two slices can be used for single pipe configurations
4748 * as additional constraint to the existing table.
4749 * However based on recent info, it should be not "pipe ratio"
4750 * but rather ratio between pixel_rate and cdclk with additional
4751 * constants, so for now we are using only table until this is
4752 * clarified. Also this is the reason why crtc_state param is
4753 * still here - we will need it once those additional constraints
4754 * pop up.
4755 */
4756 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4757}
4758
4759static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4760{
4761 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4762}
4763
4764static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4765{
4766 return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
4767}
4768
4769static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4770{
4771 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4772 enum pipe pipe = crtc->pipe;
4773
4774 if (IS_ALDERLAKE_P(dev_priv))
4775 return adlp_compute_dbuf_slices(pipe, active_pipes);
4776 else if (DISPLAY_VER(dev_priv) == 12)
4777 return tgl_compute_dbuf_slices(pipe, active_pipes);
4778 else if (DISPLAY_VER(dev_priv) == 11)
4779 return icl_compute_dbuf_slices(pipe, active_pipes);
4780 /*
4781 * For anything else just return one slice yet.
4782 * Should be extended for other platforms.
4783 */
4784 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4785}
4786
4787static u64
4788skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4789 const struct intel_plane_state *plane_state,
4790 int color_plane)
4791{
4792 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4793 const struct drm_framebuffer *fb = plane_state->hw.fb;
4794 u32 data_rate;
4795 u32 width = 0, height = 0;
4796 uint_fixed_16_16_t down_scale_amount;
4797 u64 rate;
4798
4799 if (!plane_state->uapi.visible)
4800 return 0;
4801
4802 if (plane->id == PLANE_CURSOR)
4803 return 0;
4804
4805 if (color_plane == 1 &&
4806 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4807 return 0;
4808
4809 /*
4810 * Src coordinates are already rotated by 270 degrees for
4811 * the 90/270 degree plane rotation cases (to match the
4812 * GTT mapping), hence no need to account for rotation here.
4813 */
4814 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4815 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4816
4817 /* UV plane does 1/2 pixel sub-sampling */
4818 if (color_plane == 1) {
4819 width /= 2;
4820 height /= 2;
4821 }
4822
4823 data_rate = width * height;
4824
4825 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4826
4827 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4828
4829 rate *= fb->format->cpp[color_plane];
4830 return rate;
4831}
4832
4833static u64
4834skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4835 struct intel_crtc *crtc)
4836{
4837 struct intel_crtc_state *crtc_state =
4838 intel_atomic_get_new_crtc_state(state, crtc);
4839 const struct intel_plane_state *plane_state;
4840 struct intel_plane *plane;
4841 u64 total_data_rate = 0;
4842 enum plane_id plane_id;
4843 int i;
4844
4845 /* Calculate and cache data rate for each plane */
4846 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4847 if (plane->pipe != crtc->pipe)
4848 continue;
4849
4850 plane_id = plane->id;
4851
4852 /* packed/y */
4853 crtc_state->plane_data_rate[plane_id] =
4854 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4855
4856 /* uv-plane */
4857 crtc_state->uv_plane_data_rate[plane_id] =
4858 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4859 }
4860
4861 for_each_plane_id_on_crtc(crtc, plane_id) {
4862 total_data_rate += crtc_state->plane_data_rate[plane_id];
4863 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4864 }
4865
4866 return total_data_rate;
4867}
4868
4869static u64
4870icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4871 struct intel_crtc *crtc)
4872{
4873 struct intel_crtc_state *crtc_state =
4874 intel_atomic_get_new_crtc_state(state, crtc);
4875 const struct intel_plane_state *plane_state;
4876 struct intel_plane *plane;
4877 u64 total_data_rate = 0;
4878 enum plane_id plane_id;
4879 int i;
4880
4881 /* Calculate and cache data rate for each plane */
4882 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4883 if (plane->pipe != crtc->pipe)
4884 continue;
4885
4886 plane_id = plane->id;
4887
4888 if (!plane_state->planar_linked_plane) {
4889 crtc_state->plane_data_rate[plane_id] =
4890 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4891 } else {
4892 enum plane_id y_plane_id;
4893
4894 /*
4895 * The slave plane might not iterate in
4896 * intel_atomic_crtc_state_for_each_plane_state(),
4897 * and needs the master plane state which may be
4898 * NULL if we try get_new_plane_state(), so we
4899 * always calculate from the master.
4900 */
4901 if (plane_state->planar_slave)
4902 continue;
4903
4904 /* Y plane rate is calculated on the slave */
4905 y_plane_id = plane_state->planar_linked_plane->id;
4906 crtc_state->plane_data_rate[y_plane_id] =
4907 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4908
4909 crtc_state->plane_data_rate[plane_id] =
4910 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4911 }
4912 }
4913
4914 for_each_plane_id_on_crtc(crtc, plane_id)
4915 total_data_rate += crtc_state->plane_data_rate[plane_id];
4916
4917 return total_data_rate;
4918}
4919
4920const struct skl_wm_level *
4921skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4922 enum plane_id plane_id,
4923 int level)
4924{
4925 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4926
4927 if (level == 0 && pipe_wm->use_sagv_wm)
4928 return &wm->sagv.wm0;
4929
4930 return &wm->wm[level];
4931}
4932
4933const struct skl_wm_level *
4934skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4935 enum plane_id plane_id)
4936{
4937 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4938
4939 if (pipe_wm->use_sagv_wm)
4940 return &wm->sagv.trans_wm;
4941
4942 return &wm->trans_wm;
4943}
4944
4945/*
4946 * We only disable the watermarks for each plane if
4947 * they exceed the ddb allocation of said plane. This
4948 * is done so that we don't end up touching cursor
4949 * watermarks needlessly when some other plane reduces
4950 * our max possible watermark level.
4951 *
4952 * Bspec has this to say about the PLANE_WM enable bit:
4953 * "All the watermarks at this level for all enabled
4954 * planes must be enabled before the level will be used."
4955 * So this is actually safe to do.
4956 */
4957static void
4958skl_check_wm_level(struct skl_wm_level *wm, u64 total)
4959{
4960 if (wm->min_ddb_alloc > total)
4961 memset(wm, 0, sizeof(*wm));
4962}
4963
4964static void
4965skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
4966 u64 total, u64 uv_total)
4967{
4968 if (wm->min_ddb_alloc > total ||
4969 uv_wm->min_ddb_alloc > uv_total) {
4970 memset(wm, 0, sizeof(*wm));
4971 memset(uv_wm, 0, sizeof(*uv_wm));
4972 }
4973}
4974
4975static int
4976skl_allocate_plane_ddb(struct intel_atomic_state *state,
4977 struct intel_crtc *crtc)
4978{
4979 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4980 struct intel_crtc_state *crtc_state =
4981 intel_atomic_get_new_crtc_state(state, crtc);
4982 const struct intel_dbuf_state *dbuf_state =
4983 intel_atomic_get_new_dbuf_state(state);
4984 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
4985 int num_active = hweight8(dbuf_state->active_pipes);
4986 u16 alloc_size, start = 0;
4987 u16 total[I915_MAX_PLANES] = {};
4988 u16 uv_total[I915_MAX_PLANES] = {};
4989 u64 total_data_rate;
4990 enum plane_id plane_id;
4991 u32 blocks;
4992 int level;
4993
4994 /* Clear the partitioning for disabled planes. */
4995 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4996 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4997
4998 if (!crtc_state->hw.active)
4999 return 0;
5000
5001 if (DISPLAY_VER(dev_priv) >= 11)
5002 total_data_rate =
5003 icl_get_total_relative_data_rate(state, crtc);
5004 else
5005 total_data_rate =
5006 skl_get_total_relative_data_rate(state, crtc);
5007
5008 alloc_size = skl_ddb_entry_size(alloc);
5009 if (alloc_size == 0)
5010 return 0;
5011
5012 /* Allocate fixed number of blocks for cursor. */
5013 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5014 alloc_size -= total[PLANE_CURSOR];
5015 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
5016 alloc->end - total[PLANE_CURSOR];
5017 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
5018
5019 if (total_data_rate == 0)
5020 return 0;
5021
5022 /*
5023 * Find the highest watermark level for which we can satisfy the block
5024 * requirement of active planes.
5025 */
5026 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5027 blocks = 0;
5028 for_each_plane_id_on_crtc(crtc, plane_id) {
5029 const struct skl_plane_wm *wm =
5030 &crtc_state->wm.skl.optimal.planes[plane_id];
5031
5032 if (plane_id == PLANE_CURSOR) {
5033 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
5034 drm_WARN_ON(&dev_priv->drm,
5035 wm->wm[level].min_ddb_alloc != U16_MAX);
5036 blocks = U32_MAX;
5037 break;
5038 }
5039 continue;
5040 }
5041
5042 blocks += wm->wm[level].min_ddb_alloc;
5043 blocks += wm->uv_wm[level].min_ddb_alloc;
5044 }
5045
5046 if (blocks <= alloc_size) {
5047 alloc_size -= blocks;
5048 break;
5049 }
5050 }
5051
5052 if (level < 0) {
5053 drm_dbg_kms(&dev_priv->drm,
5054 "Requested display configuration exceeds system DDB limitations");
5055 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5056 blocks, alloc_size);
5057 return -EINVAL;
5058 }
5059
5060 /*
5061 * Grant each plane the blocks it requires at the highest achievable
5062 * watermark level, plus an extra share of the leftover blocks
5063 * proportional to its relative data rate.
5064 */
5065 for_each_plane_id_on_crtc(crtc, plane_id) {
5066 const struct skl_plane_wm *wm =
5067 &crtc_state->wm.skl.optimal.planes[plane_id];
5068 u64 rate;
5069 u16 extra;
5070
5071 if (plane_id == PLANE_CURSOR)
5072 continue;
5073
5074 /*
5075 * We've accounted for all active planes; remaining planes are
5076 * all disabled.
5077 */
5078 if (total_data_rate == 0)
5079 break;
5080
5081 rate = crtc_state->plane_data_rate[plane_id];
5082 extra = min_t(u16, alloc_size,
5083 DIV64_U64_ROUND_UP(alloc_size * rate,
5084 total_data_rate));
5085 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
5086 alloc_size -= extra;
5087 total_data_rate -= rate;
5088
5089 if (total_data_rate == 0)
5090 break;
5091
5092 rate = crtc_state->uv_plane_data_rate[plane_id];
5093 extra = min_t(u16, alloc_size,
5094 DIV64_U64_ROUND_UP(alloc_size * rate,
5095 total_data_rate));
5096 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
5097 alloc_size -= extra;
5098 total_data_rate -= rate;
5099 }
5100 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
5101
5102 /* Set the actual DDB start/end points for each plane */
5103 start = alloc->start;
5104 for_each_plane_id_on_crtc(crtc, plane_id) {
5105 struct skl_ddb_entry *plane_alloc =
5106 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5107 struct skl_ddb_entry *uv_plane_alloc =
5108 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5109
5110 if (plane_id == PLANE_CURSOR)
5111 continue;
5112
5113 /* Gen11+ uses a separate plane for UV watermarks */
5114 drm_WARN_ON(&dev_priv->drm,
5115 DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
5116
5117 /* Leave disabled planes at (0,0) */
5118 if (total[plane_id]) {
5119 plane_alloc->start = start;
5120 start += total[plane_id];
5121 plane_alloc->end = start;
5122 }
5123
5124 if (uv_total[plane_id]) {
5125 uv_plane_alloc->start = start;
5126 start += uv_total[plane_id];
5127 uv_plane_alloc->end = start;
5128 }
5129 }
5130
5131 /*
5132 * When we calculated watermark values we didn't know how high
5133 * of a level we'd actually be able to hit, so we just marked
5134 * all levels as "enabled." Go back now and disable the ones
5135 * that aren't actually possible.
5136 */
5137 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5138 for_each_plane_id_on_crtc(crtc, plane_id) {
5139 struct skl_plane_wm *wm =
5140 &crtc_state->wm.skl.optimal.planes[plane_id];
5141
5142 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5143 total[plane_id], uv_total[plane_id]);
5144
5145 /*
5146 * Wa_1408961008:icl, ehl
5147 * Underruns with WM1+ disabled
5148 */
5149 if (DISPLAY_VER(dev_priv) == 11 &&
5150 level == 1 && wm->wm[0].enable) {
5151 wm->wm[level].blocks = wm->wm[0].blocks;
5152 wm->wm[level].lines = wm->wm[0].lines;
5153 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5154 }
5155 }
5156 }
5157
5158 /*
5159 * Go back and disable the transition and SAGV watermarks
5160 * if it turns out we don't have enough DDB blocks for them.
5161 */
5162 for_each_plane_id_on_crtc(crtc, plane_id) {
5163 struct skl_plane_wm *wm =
5164 &crtc_state->wm.skl.optimal.planes[plane_id];
5165
5166 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5167 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5168 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5169 }
5170
5171 return 0;
5172}
5173
5174/*
5175 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5176 * for the read latency) and cpp should always be <= 8, so that
5177 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5178 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5179*/
5180static uint_fixed_16_16_t
5181skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5182 u8 cpp, u32 latency, u32 dbuf_block_size)
5183{
5184 u32 wm_intermediate_val;
5185 uint_fixed_16_16_t ret;
5186
5187 if (latency == 0)
5188 return FP_16_16_MAX;
5189
5190 wm_intermediate_val = latency * pixel_rate * cpp;
5191 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5192
5193 if (DISPLAY_VER(dev_priv) >= 10)
5194 ret = add_fixed16_u32(ret, 1);
5195
5196 return ret;
5197}
5198
5199static uint_fixed_16_16_t
5200skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5201 uint_fixed_16_16_t plane_blocks_per_line)
5202{
5203 u32 wm_intermediate_val;
5204 uint_fixed_16_16_t ret;
5205
5206 if (latency == 0)
5207 return FP_16_16_MAX;
5208
5209 wm_intermediate_val = latency * pixel_rate;
5210 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5211 pipe_htotal * 1000);
5212 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5213 return ret;
5214}
5215
5216static uint_fixed_16_16_t
5217intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5218{
5219 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5220 u32 pixel_rate;
5221 u32 crtc_htotal;
5222 uint_fixed_16_16_t linetime_us;
5223
5224 if (!crtc_state->hw.active)
5225 return u32_to_fixed16(0);
5226
5227 pixel_rate = crtc_state->pixel_rate;
5228
5229 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5230 return u32_to_fixed16(0);
5231
5232 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5233 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5234
5235 return linetime_us;
5236}
5237
5238static int
5239skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5240 int width, const struct drm_format_info *format,
5241 u64 modifier, unsigned int rotation,
5242 u32 plane_pixel_rate, struct skl_wm_params *wp,
5243 int color_plane)
5244{
5245 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5246 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5247 u32 interm_pbpl;
5248
5249 /* only planar format has two planes */
5250 if (color_plane == 1 &&
5251 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5252 drm_dbg_kms(&dev_priv->drm,
5253 "Non planar format have single plane\n");
5254 return -EINVAL;
5255 }
5256
5257 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5258 modifier == I915_FORMAT_MOD_Yf_TILED ||
5259 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5260 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5261 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5262 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5263 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5264 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5265
5266 wp->width = width;
5267 if (color_plane == 1 && wp->is_planar)
5268 wp->width /= 2;
5269
5270 wp->cpp = format->cpp[color_plane];
5271 wp->plane_pixel_rate = plane_pixel_rate;
5272
5273 if (DISPLAY_VER(dev_priv) >= 11 &&
5274 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5275 wp->dbuf_block_size = 256;
5276 else
5277 wp->dbuf_block_size = 512;
5278
5279 if (drm_rotation_90_or_270(rotation)) {
5280 switch (wp->cpp) {
5281 case 1:
5282 wp->y_min_scanlines = 16;
5283 break;
5284 case 2:
5285 wp->y_min_scanlines = 8;
5286 break;
5287 case 4:
5288 wp->y_min_scanlines = 4;
5289 break;
5290 default:
5291 MISSING_CASE(wp->cpp);
5292 return -EINVAL;
5293 }
5294 } else {
5295 wp->y_min_scanlines = 4;
5296 }
5297
5298 if (skl_needs_memory_bw_wa(dev_priv))
5299 wp->y_min_scanlines *= 2;
5300
5301 wp->plane_bytes_per_line = wp->width * wp->cpp;
5302 if (wp->y_tiled) {
5303 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5304 wp->y_min_scanlines,
5305 wp->dbuf_block_size);
5306
5307 if (DISPLAY_VER(dev_priv) >= 10)
5308 interm_pbpl++;
5309
5310 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5311 wp->y_min_scanlines);
5312 } else {
5313 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5314 wp->dbuf_block_size);
5315
5316 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5317 interm_pbpl++;
5318
5319 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5320 }
5321
5322 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5323 wp->plane_blocks_per_line);
5324
5325 wp->linetime_us = fixed16_to_u32_round_up(
5326 intel_get_linetime_us(crtc_state));
5327
5328 return 0;
5329}
5330
5331static int
5332skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5333 const struct intel_plane_state *plane_state,
5334 struct skl_wm_params *wp, int color_plane)
5335{
5336 const struct drm_framebuffer *fb = plane_state->hw.fb;
5337 int width;
5338
5339 /*
5340 * Src coordinates are already rotated by 270 degrees for
5341 * the 90/270 degree plane rotation cases (to match the
5342 * GTT mapping), hence no need to account for rotation here.
5343 */
5344 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5345
5346 return skl_compute_wm_params(crtc_state, width,
5347 fb->format, fb->modifier,
5348 plane_state->hw.rotation,
5349 intel_plane_pixel_rate(crtc_state, plane_state),
5350 wp, color_plane);
5351}
5352
5353static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5354{
5355 if (DISPLAY_VER(dev_priv) >= 10)
5356 return true;
5357
5358 /* The number of lines are ignored for the level 0 watermark. */
5359 return level > 0;
5360}
5361
5362static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5363{
5364 if (DISPLAY_VER(dev_priv) >= 13)
5365 return 255;
5366 else
5367 return 31;
5368}
5369
5370static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5371 int level,
5372 unsigned int latency,
5373 const struct skl_wm_params *wp,
5374 const struct skl_wm_level *result_prev,
5375 struct skl_wm_level *result /* out */)
5376{
5377 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5378 uint_fixed_16_16_t method1, method2;
5379 uint_fixed_16_16_t selected_result;
5380 u32 blocks, lines, min_ddb_alloc = 0;
5381
5382 if (latency == 0) {
5383 /* reject it */
5384 result->min_ddb_alloc = U16_MAX;
5385 return;
5386 }
5387
5388 /*
5389 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5390 * Display WA #1141: kbl,cfl
5391 */
5392 if ((IS_KABYLAKE(dev_priv) ||
5393 IS_COFFEELAKE(dev_priv) ||
5394 IS_COMETLAKE(dev_priv)) &&
5395 dev_priv->ipc_enabled)
5396 latency += 4;
5397
5398 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5399 latency += 15;
5400
5401 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5402 wp->cpp, latency, wp->dbuf_block_size);
5403 method2 = skl_wm_method2(wp->plane_pixel_rate,
5404 crtc_state->hw.pipe_mode.crtc_htotal,
5405 latency,
5406 wp->plane_blocks_per_line);
5407
5408 if (wp->y_tiled) {
5409 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5410 } else {
5411 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5412 wp->dbuf_block_size < 1) &&
5413 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5414 selected_result = method2;
5415 } else if (latency >= wp->linetime_us) {
5416 if (DISPLAY_VER(dev_priv) == 9)
5417 selected_result = min_fixed16(method1, method2);
5418 else
5419 selected_result = method2;
5420 } else {
5421 selected_result = method1;
5422 }
5423 }
5424
5425 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5426 lines = div_round_up_fixed16(selected_result,
5427 wp->plane_blocks_per_line);
5428
5429 if (DISPLAY_VER(dev_priv) == 9) {
5430 /* Display WA #1125: skl,bxt,kbl */
5431 if (level == 0 && wp->rc_surface)
5432 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5433
5434 /* Display WA #1126: skl,bxt,kbl */
5435 if (level >= 1 && level <= 7) {
5436 if (wp->y_tiled) {
5437 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5438 lines += wp->y_min_scanlines;
5439 } else {
5440 blocks++;
5441 }
5442
5443 /*
5444 * Make sure result blocks for higher latency levels are
5445 * atleast as high as level below the current level.
5446 * Assumption in DDB algorithm optimization for special
5447 * cases. Also covers Display WA #1125 for RC.
5448 */
5449 if (result_prev->blocks > blocks)
5450 blocks = result_prev->blocks;
5451 }
5452 }
5453
5454 if (DISPLAY_VER(dev_priv) >= 11) {
5455 if (wp->y_tiled) {
5456 int extra_lines;
5457
5458 if (lines % wp->y_min_scanlines == 0)
5459 extra_lines = wp->y_min_scanlines;
5460 else
5461 extra_lines = wp->y_min_scanlines * 2 -
5462 lines % wp->y_min_scanlines;
5463
5464 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5465 wp->plane_blocks_per_line);
5466 } else {
5467 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5468 }
5469 }
5470
5471 if (!skl_wm_has_lines(dev_priv, level))
5472 lines = 0;
5473
5474 if (lines > skl_wm_max_lines(dev_priv)) {
5475 /* reject it */
5476 result->min_ddb_alloc = U16_MAX;
5477 return;
5478 }
5479
5480 /*
5481 * If lines is valid, assume we can use this watermark level
5482 * for now. We'll come back and disable it after we calculate the
5483 * DDB allocation if it turns out we don't actually have enough
5484 * blocks to satisfy it.
5485 */
5486 result->blocks = blocks;
5487 result->lines = lines;
5488 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5489 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5490 result->enable = true;
5491
5492 if (DISPLAY_VER(dev_priv) < 12)
5493 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5494}
5495
5496static void
5497skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5498 const struct skl_wm_params *wm_params,
5499 struct skl_wm_level *levels)
5500{
5501 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5502 int level, max_level = ilk_wm_max_level(dev_priv);
5503 struct skl_wm_level *result_prev = &levels[0];
5504
5505 for (level = 0; level <= max_level; level++) {
5506 struct skl_wm_level *result = &levels[level];
5507 unsigned int latency = dev_priv->wm.skl_latency[level];
5508
5509 skl_compute_plane_wm(crtc_state, level, latency,
5510 wm_params, result_prev, result);
5511
5512 result_prev = result;
5513 }
5514}
5515
5516static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5517 const struct skl_wm_params *wm_params,
5518 struct skl_plane_wm *plane_wm)
5519{
5520 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5521 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5522 struct skl_wm_level *levels = plane_wm->wm;
5523 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5524
5525 skl_compute_plane_wm(crtc_state, 0, latency,
5526 wm_params, &levels[0],
5527 sagv_wm);
5528}
5529
5530static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5531 struct skl_wm_level *trans_wm,
5532 const struct skl_wm_level *wm0,
5533 const struct skl_wm_params *wp)
5534{
5535 u16 trans_min, trans_amount, trans_y_tile_min;
5536 u16 wm0_blocks, trans_offset, blocks;
5537
5538 /* Transition WM don't make any sense if ipc is disabled */
5539 if (!dev_priv->ipc_enabled)
5540 return;
5541
5542 /*
5543 * WaDisableTWM:skl,kbl,cfl,bxt
5544 * Transition WM are not recommended by HW team for GEN9
5545 */
5546 if (DISPLAY_VER(dev_priv) == 9)
5547 return;
5548
5549 if (DISPLAY_VER(dev_priv) >= 11)
5550 trans_min = 4;
5551 else
5552 trans_min = 14;
5553
5554 /* Display WA #1140: glk,cnl */
5555 if (DISPLAY_VER(dev_priv) == 10)
5556 trans_amount = 0;
5557 else
5558 trans_amount = 10; /* This is configurable amount */
5559
5560 trans_offset = trans_min + trans_amount;
5561
5562 /*
5563 * The spec asks for Selected Result Blocks for wm0 (the real value),
5564 * not Result Blocks (the integer value). Pay attention to the capital
5565 * letters. The value wm_l0->blocks is actually Result Blocks, but
5566 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5567 * and since we later will have to get the ceiling of the sum in the
5568 * transition watermarks calculation, we can just pretend Selected
5569 * Result Blocks is Result Blocks minus 1 and it should work for the
5570 * current platforms.
5571 */
5572 wm0_blocks = wm0->blocks - 1;
5573
5574 if (wp->y_tiled) {
5575 trans_y_tile_min =
5576 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5577 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5578 } else {
5579 blocks = wm0_blocks + trans_offset;
5580 }
5581 blocks++;
5582
5583 /*
5584 * Just assume we can enable the transition watermark. After
5585 * computing the DDB we'll come back and disable it if that
5586 * assumption turns out to be false.
5587 */
5588 trans_wm->blocks = blocks;
5589 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5590 trans_wm->enable = true;
5591}
5592
5593static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5594 const struct intel_plane_state *plane_state,
5595 enum plane_id plane_id, int color_plane)
5596{
5597 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5598 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5599 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5600 struct skl_wm_params wm_params;
5601 int ret;
5602
5603 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5604 &wm_params, color_plane);
5605 if (ret)
5606 return ret;
5607
5608 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5609
5610 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5611 &wm->wm[0], &wm_params);
5612
5613 if (DISPLAY_VER(dev_priv) >= 12) {
5614 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5615
5616 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5617 &wm->sagv.wm0, &wm_params);
5618 }
5619
5620 return 0;
5621}
5622
5623static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5624 const struct intel_plane_state *plane_state,
5625 enum plane_id plane_id)
5626{
5627 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5628 struct skl_wm_params wm_params;
5629 int ret;
5630
5631 wm->is_planar = true;
5632
5633 /* uv plane watermarks must also be validated for NV12/Planar */
5634 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5635 &wm_params, 1);
5636 if (ret)
5637 return ret;
5638
5639 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5640
5641 return 0;
5642}
5643
5644static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5645 const struct intel_plane_state *plane_state)
5646{
5647 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5648 enum plane_id plane_id = plane->id;
5649 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5650 const struct drm_framebuffer *fb = plane_state->hw.fb;
5651 int ret;
5652
5653 memset(wm, 0, sizeof(*wm));
5654
5655 if (!intel_wm_plane_visible(crtc_state, plane_state))
5656 return 0;
5657
5658 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5659 plane_id, 0);
5660 if (ret)
5661 return ret;
5662
5663 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5664 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5665 plane_id);
5666 if (ret)
5667 return ret;
5668 }
5669
5670 return 0;
5671}
5672
5673static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5674 const struct intel_plane_state *plane_state)
5675{
5676 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5677 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5678 enum plane_id plane_id = plane->id;
5679 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5680 int ret;
5681
5682 /* Watermarks calculated in master */
5683 if (plane_state->planar_slave)
5684 return 0;
5685
5686 memset(wm, 0, sizeof(*wm));
5687
5688 if (plane_state->planar_linked_plane) {
5689 const struct drm_framebuffer *fb = plane_state->hw.fb;
5690 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5691
5692 drm_WARN_ON(&dev_priv->drm,
5693 !intel_wm_plane_visible(crtc_state, plane_state));
5694 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5695 fb->format->num_planes == 1);
5696
5697 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5698 y_plane_id, 0);
5699 if (ret)
5700 return ret;
5701
5702 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5703 plane_id, 1);
5704 if (ret)
5705 return ret;
5706 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5707 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5708 plane_id, 0);
5709 if (ret)
5710 return ret;
5711 }
5712
5713 return 0;
5714}
5715
5716static int skl_build_pipe_wm(struct intel_atomic_state *state,
5717 struct intel_crtc *crtc)
5718{
5719 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5720 struct intel_crtc_state *crtc_state =
5721 intel_atomic_get_new_crtc_state(state, crtc);
5722 const struct intel_plane_state *plane_state;
5723 struct intel_plane *plane;
5724 int ret, i;
5725
5726 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5727 /*
5728 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5729 * instead but we don't populate that correctly for NV12 Y
5730 * planes so for now hack this.
5731 */
5732 if (plane->pipe != crtc->pipe)
5733 continue;
5734
5735 if (DISPLAY_VER(dev_priv) >= 11)
5736 ret = icl_build_plane_wm(crtc_state, plane_state);
5737 else
5738 ret = skl_build_plane_wm(crtc_state, plane_state);
5739 if (ret)
5740 return ret;
5741 }
5742
5743 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5744
5745 return 0;
5746}
5747
5748static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5749 i915_reg_t reg,
5750 const struct skl_ddb_entry *entry)
5751{
5752 if (entry->end)
5753 intel_de_write_fw(dev_priv, reg,
5754 (entry->end - 1) << 16 | entry->start);
5755 else
5756 intel_de_write_fw(dev_priv, reg, 0);
5757}
5758
5759static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5760 i915_reg_t reg,
5761 const struct skl_wm_level *level)
5762{
5763 u32 val = 0;
5764
5765 if (level->enable)
5766 val |= PLANE_WM_EN;
5767 if (level->ignore_lines)
5768 val |= PLANE_WM_IGNORE_LINES;
5769 val |= level->blocks;
5770 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5771
5772 intel_de_write_fw(dev_priv, reg, val);
5773}
5774
5775void skl_write_plane_wm(struct intel_plane *plane,
5776 const struct intel_crtc_state *crtc_state)
5777{
5778 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5779 int level, max_level = ilk_wm_max_level(dev_priv);
5780 enum plane_id plane_id = plane->id;
5781 enum pipe pipe = plane->pipe;
5782 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5783 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5784 const struct skl_ddb_entry *ddb_y =
5785 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5786 const struct skl_ddb_entry *ddb_uv =
5787 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5788
5789 for (level = 0; level <= max_level; level++)
5790 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5791 skl_plane_wm_level(pipe_wm, plane_id, level));
5792
5793 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5794 skl_plane_trans_wm(pipe_wm, plane_id));
5795
5796 if (HAS_HW_SAGV_WM(dev_priv)) {
5797 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5798 &wm->sagv.wm0);
5799 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5800 &wm->sagv.trans_wm);
5801 }
5802
5803 if (DISPLAY_VER(dev_priv) >= 11) {
5804 skl_ddb_entry_write(dev_priv,
5805 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5806 return;
5807 }
5808
5809 if (wm->is_planar)
5810 swap(ddb_y, ddb_uv);
5811
5812 skl_ddb_entry_write(dev_priv,
5813 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5814 skl_ddb_entry_write(dev_priv,
5815 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5816}
5817
5818void skl_write_cursor_wm(struct intel_plane *plane,
5819 const struct intel_crtc_state *crtc_state)
5820{
5821 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5822 int level, max_level = ilk_wm_max_level(dev_priv);
5823 enum plane_id plane_id = plane->id;
5824 enum pipe pipe = plane->pipe;
5825 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5826 const struct skl_ddb_entry *ddb =
5827 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5828
5829 for (level = 0; level <= max_level; level++)
5830 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5831 skl_plane_wm_level(pipe_wm, plane_id, level));
5832
5833 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5834 skl_plane_trans_wm(pipe_wm, plane_id));
5835
5836 if (HAS_HW_SAGV_WM(dev_priv)) {
5837 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5838
5839 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5840 &wm->sagv.wm0);
5841 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5842 &wm->sagv.trans_wm);
5843 }
5844
5845 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5846}
5847
5848bool skl_wm_level_equals(const struct skl_wm_level *l1,
5849 const struct skl_wm_level *l2)
5850{
5851 return l1->enable == l2->enable &&
5852 l1->ignore_lines == l2->ignore_lines &&
5853 l1->lines == l2->lines &&
5854 l1->blocks == l2->blocks;
5855}
5856
5857static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5858 const struct skl_plane_wm *wm1,
5859 const struct skl_plane_wm *wm2)
5860{
5861 int level, max_level = ilk_wm_max_level(dev_priv);
5862
5863 for (level = 0; level <= max_level; level++) {
5864 /*
5865 * We don't check uv_wm as the hardware doesn't actually
5866 * use it. It only gets used for calculating the required
5867 * ddb allocation.
5868 */
5869 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5870 return false;
5871 }
5872
5873 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5874 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5875 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5876}
5877
5878static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5879 const struct skl_ddb_entry *b)
5880{
5881 return a->start < b->end && b->start < a->end;
5882}
5883
5884static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5885 const struct skl_ddb_entry *b)
5886{
5887 if (a->end && b->end) {
5888 a->start = min(a->start, b->start);
5889 a->end = max(a->end, b->end);
5890 } else if (b->end) {
5891 a->start = b->start;
5892 a->end = b->end;
5893 }
5894}
5895
5896bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5897 const struct skl_ddb_entry *entries,
5898 int num_entries, int ignore_idx)
5899{
5900 int i;
5901
5902 for (i = 0; i < num_entries; i++) {
5903 if (i != ignore_idx &&
5904 skl_ddb_entries_overlap(ddb, &entries[i]))
5905 return true;
5906 }
5907
5908 return false;
5909}
5910
5911static int
5912skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5913 struct intel_crtc_state *new_crtc_state)
5914{
5915 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5916 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5917 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5918 struct intel_plane *plane;
5919
5920 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5921 struct intel_plane_state *plane_state;
5922 enum plane_id plane_id = plane->id;
5923
5924 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5925 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5926 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5927 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5928 continue;
5929
5930 plane_state = intel_atomic_get_plane_state(state, plane);
5931 if (IS_ERR(plane_state))
5932 return PTR_ERR(plane_state);
5933
5934 new_crtc_state->update_planes |= BIT(plane_id);
5935 }
5936
5937 return 0;
5938}
5939
5940static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
5941{
5942 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
5943 u8 enabled_slices;
5944 enum pipe pipe;
5945
5946 /*
5947 * FIXME: For now we always enable slice S1 as per
5948 * the Bspec display initialization sequence.
5949 */
5950 enabled_slices = BIT(DBUF_S1);
5951
5952 for_each_pipe(dev_priv, pipe)
5953 enabled_slices |= dbuf_state->slices[pipe];
5954
5955 return enabled_slices;
5956}
5957
5958static int
5959skl_compute_ddb(struct intel_atomic_state *state)
5960{
5961 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5962 const struct intel_dbuf_state *old_dbuf_state;
5963 struct intel_dbuf_state *new_dbuf_state = NULL;
5964 const struct intel_crtc_state *old_crtc_state;
5965 struct intel_crtc_state *new_crtc_state;
5966 struct intel_crtc *crtc;
5967 int ret, i;
5968
5969 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5970 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5971 if (IS_ERR(new_dbuf_state))
5972 return PTR_ERR(new_dbuf_state);
5973
5974 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5975 break;
5976 }
5977
5978 if (!new_dbuf_state)
5979 return 0;
5980
5981 new_dbuf_state->active_pipes =
5982 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
5983
5984 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
5985 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5986 if (ret)
5987 return ret;
5988 }
5989
5990 for_each_intel_crtc(&dev_priv->drm, crtc) {
5991 enum pipe pipe = crtc->pipe;
5992
5993 new_dbuf_state->slices[pipe] =
5994 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
5995
5996 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
5997 continue;
5998
5999 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6000 if (ret)
6001 return ret;
6002 }
6003
6004 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6005
6006 if (IS_ALDERLAKE_P(dev_priv))
6007 new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6008
6009 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6010 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6011 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6012 if (ret)
6013 return ret;
6014
6015 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6016 /* TODO: Implement vblank synchronized MBUS joining changes */
6017 ret = intel_modeset_all_pipes(state);
6018 if (ret)
6019 return ret;
6020 }
6021
6022 drm_dbg_kms(&dev_priv->drm,
6023 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6024 old_dbuf_state->enabled_slices,
6025 new_dbuf_state->enabled_slices,
6026 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6027 yesno(old_dbuf_state->joined_mbus),
6028 yesno(new_dbuf_state->joined_mbus));
6029 }
6030
6031 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6032 enum pipe pipe = crtc->pipe;
6033
6034 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6035
6036 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6037 continue;
6038
6039 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6040 if (ret)
6041 return ret;
6042 }
6043
6044 for_each_intel_crtc(&dev_priv->drm, crtc) {
6045 ret = skl_crtc_allocate_ddb(state, crtc);
6046 if (ret)
6047 return ret;
6048 }
6049
6050 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6051 new_crtc_state, i) {
6052 ret = skl_allocate_plane_ddb(state, crtc);
6053 if (ret)
6054 return ret;
6055
6056 ret = skl_ddb_add_affected_planes(old_crtc_state,
6057 new_crtc_state);
6058 if (ret)
6059 return ret;
6060 }
6061
6062 return 0;
6063}
6064
6065static char enast(bool enable)
6066{
6067 return enable ? '*' : ' ';
6068}
6069
6070static void
6071skl_print_wm_changes(struct intel_atomic_state *state)
6072{
6073 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6074 const struct intel_crtc_state *old_crtc_state;
6075 const struct intel_crtc_state *new_crtc_state;
6076 struct intel_plane *plane;
6077 struct intel_crtc *crtc;
6078 int i;
6079
6080 if (!drm_debug_enabled(DRM_UT_KMS))
6081 return;
6082
6083 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6084 new_crtc_state, i) {
6085 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6086
6087 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6088 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6089
6090 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6091 enum plane_id plane_id = plane->id;
6092 const struct skl_ddb_entry *old, *new;
6093
6094 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6095 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6096
6097 if (skl_ddb_entry_equal(old, new))
6098 continue;
6099
6100 drm_dbg_kms(&dev_priv->drm,
6101 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6102 plane->base.base.id, plane->base.name,
6103 old->start, old->end, new->start, new->end,
6104 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6105 }
6106
6107 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6108 enum plane_id plane_id = plane->id;
6109 const struct skl_plane_wm *old_wm, *new_wm;
6110
6111 old_wm = &old_pipe_wm->planes[plane_id];
6112 new_wm = &new_pipe_wm->planes[plane_id];
6113
6114 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6115 continue;
6116
6117 drm_dbg_kms(&dev_priv->drm,
6118 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6119 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6120 plane->base.base.id, plane->base.name,
6121 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6122 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6123 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6124 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6125 enast(old_wm->trans_wm.enable),
6126 enast(old_wm->sagv.wm0.enable),
6127 enast(old_wm->sagv.trans_wm.enable),
6128 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6129 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6130 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6131 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6132 enast(new_wm->trans_wm.enable),
6133 enast(new_wm->sagv.wm0.enable),
6134 enast(new_wm->sagv.trans_wm.enable));
6135
6136 drm_dbg_kms(&dev_priv->drm,
6137 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6138 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6139 plane->base.base.id, plane->base.name,
6140 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6141 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6142 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6143 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6144 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6145 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6146 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6147 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6148 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6149 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6150 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6151 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6152 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6153 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6154 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6155 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6156 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6157 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6158 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6159 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6160 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6161 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6162
6163 drm_dbg_kms(&dev_priv->drm,
6164 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6165 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6166 plane->base.base.id, plane->base.name,
6167 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6168 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6169 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6170 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6171 old_wm->trans_wm.blocks,
6172 old_wm->sagv.wm0.blocks,
6173 old_wm->sagv.trans_wm.blocks,
6174 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6175 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6176 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6177 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6178 new_wm->trans_wm.blocks,
6179 new_wm->sagv.wm0.blocks,
6180 new_wm->sagv.trans_wm.blocks);
6181
6182 drm_dbg_kms(&dev_priv->drm,
6183 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6184 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6185 plane->base.base.id, plane->base.name,
6186 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6187 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6188 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6189 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6190 old_wm->trans_wm.min_ddb_alloc,
6191 old_wm->sagv.wm0.min_ddb_alloc,
6192 old_wm->sagv.trans_wm.min_ddb_alloc,
6193 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6194 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6195 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6196 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6197 new_wm->trans_wm.min_ddb_alloc,
6198 new_wm->sagv.wm0.min_ddb_alloc,
6199 new_wm->sagv.trans_wm.min_ddb_alloc);
6200 }
6201 }
6202}
6203
6204static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6205 const struct skl_pipe_wm *old_pipe_wm,
6206 const struct skl_pipe_wm *new_pipe_wm)
6207{
6208 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6209 int level, max_level = ilk_wm_max_level(i915);
6210
6211 for (level = 0; level <= max_level; level++) {
6212 /*
6213 * We don't check uv_wm as the hardware doesn't actually
6214 * use it. It only gets used for calculating the required
6215 * ddb allocation.
6216 */
6217 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6218 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6219 return false;
6220 }
6221
6222 if (HAS_HW_SAGV_WM(i915)) {
6223 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6224 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6225
6226 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6227 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6228 return false;
6229 }
6230
6231 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6232 skl_plane_trans_wm(new_pipe_wm, plane->id));
6233}
6234
6235/*
6236 * To make sure the cursor watermark registers are always consistent
6237 * with our computed state the following scenario needs special
6238 * treatment:
6239 *
6240 * 1. enable cursor
6241 * 2. move cursor entirely offscreen
6242 * 3. disable cursor
6243 *
6244 * Step 2. does call .disable_plane() but does not zero the watermarks
6245 * (since we consider an offscreen cursor still active for the purposes
6246 * of watermarks). Step 3. would not normally call .disable_plane()
6247 * because the actual plane visibility isn't changing, and we don't
6248 * deallocate the cursor ddb until the pipe gets disabled. So we must
6249 * force step 3. to call .disable_plane() to update the watermark
6250 * registers properly.
6251 *
6252 * Other planes do not suffer from this issues as their watermarks are
6253 * calculated based on the actual plane visibility. The only time this
6254 * can trigger for the other planes is during the initial readout as the
6255 * default value of the watermarks registers is not zero.
6256 */
6257static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6258 struct intel_crtc *crtc)
6259{
6260 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6261 const struct intel_crtc_state *old_crtc_state =
6262 intel_atomic_get_old_crtc_state(state, crtc);
6263 struct intel_crtc_state *new_crtc_state =
6264 intel_atomic_get_new_crtc_state(state, crtc);
6265 struct intel_plane *plane;
6266
6267 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6268 struct intel_plane_state *plane_state;
6269 enum plane_id plane_id = plane->id;
6270
6271 /*
6272 * Force a full wm update for every plane on modeset.
6273 * Required because the reset value of the wm registers
6274 * is non-zero, whereas we want all disabled planes to
6275 * have zero watermarks. So if we turn off the relevant
6276 * power well the hardware state will go out of sync
6277 * with the software state.
6278 */
6279 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6280 skl_plane_selected_wm_equals(plane,
6281 &old_crtc_state->wm.skl.optimal,
6282 &new_crtc_state->wm.skl.optimal))
6283 continue;
6284
6285 plane_state = intel_atomic_get_plane_state(state, plane);
6286 if (IS_ERR(plane_state))
6287 return PTR_ERR(plane_state);
6288
6289 new_crtc_state->update_planes |= BIT(plane_id);
6290 }
6291
6292 return 0;
6293}
6294
6295static int
6296skl_compute_wm(struct intel_atomic_state *state)
6297{
6298 struct intel_crtc *crtc;
6299 struct intel_crtc_state *new_crtc_state;
6300 int ret, i;
6301
6302 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6303 ret = skl_build_pipe_wm(state, crtc);
6304 if (ret)
6305 return ret;
6306 }
6307
6308 ret = skl_compute_ddb(state);
6309 if (ret)
6310 return ret;
6311
6312 ret = intel_compute_sagv_mask(state);
6313 if (ret)
6314 return ret;
6315
6316 /*
6317 * skl_compute_ddb() will have adjusted the final watermarks
6318 * based on how much ddb is available. Now we can actually
6319 * check if the final watermarks changed.
6320 */
6321 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6322 ret = skl_wm_add_affected_planes(state, crtc);
6323 if (ret)
6324 return ret;
6325 }
6326
6327 skl_print_wm_changes(state);
6328
6329 return 0;
6330}
6331
6332static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6333 struct intel_wm_config *config)
6334{
6335 struct intel_crtc *crtc;
6336
6337 /* Compute the currently _active_ config */
6338 for_each_intel_crtc(&dev_priv->drm, crtc) {
6339 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6340
6341 if (!wm->pipe_enabled)
6342 continue;
6343
6344 config->sprites_enabled |= wm->sprites_enabled;
6345 config->sprites_scaled |= wm->sprites_scaled;
6346 config->num_pipes_active++;
6347 }
6348}
6349
6350static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6351{
6352 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6353 struct ilk_wm_maximums max;
6354 struct intel_wm_config config = {};
6355 struct ilk_wm_values results = {};
6356 enum intel_ddb_partitioning partitioning;
6357
6358 ilk_compute_wm_config(dev_priv, &config);
6359
6360 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6361 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6362
6363 /* 5/6 split only in single pipe config on IVB+ */
6364 if (DISPLAY_VER(dev_priv) >= 7 &&
6365 config.num_pipes_active == 1 && config.sprites_enabled) {
6366 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6367 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6368
6369 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6370 } else {
6371 best_lp_wm = &lp_wm_1_2;
6372 }
6373
6374 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6375 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6376
6377 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6378
6379 ilk_write_wm_values(dev_priv, &results);
6380}
6381
6382static void ilk_initial_watermarks(struct intel_atomic_state *state,
6383 struct intel_crtc *crtc)
6384{
6385 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6386 const struct intel_crtc_state *crtc_state =
6387 intel_atomic_get_new_crtc_state(state, crtc);
6388
6389 mutex_lock(&dev_priv->wm.wm_mutex);
6390 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6391 ilk_program_watermarks(dev_priv);
6392 mutex_unlock(&dev_priv->wm.wm_mutex);
6393}
6394
6395static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6396 struct intel_crtc *crtc)
6397{
6398 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6399 const struct intel_crtc_state *crtc_state =
6400 intel_atomic_get_new_crtc_state(state, crtc);
6401
6402 if (!crtc_state->wm.need_postvbl_update)
6403 return;
6404
6405 mutex_lock(&dev_priv->wm.wm_mutex);
6406 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6407 ilk_program_watermarks(dev_priv);
6408 mutex_unlock(&dev_priv->wm.wm_mutex);
6409}
6410
6411static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6412{
6413 level->enable = val & PLANE_WM_EN;
6414 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6415 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6416 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6417}
6418
6419void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6420 struct skl_pipe_wm *out)
6421{
6422 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6423 enum pipe pipe = crtc->pipe;
6424 int level, max_level;
6425 enum plane_id plane_id;
6426 u32 val;
6427
6428 max_level = ilk_wm_max_level(dev_priv);
6429
6430 for_each_plane_id_on_crtc(crtc, plane_id) {
6431 struct skl_plane_wm *wm = &out->planes[plane_id];
6432
6433 for (level = 0; level <= max_level; level++) {
6434 if (plane_id != PLANE_CURSOR)
6435 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6436 else
6437 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6438
6439 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6440 }
6441
6442 if (plane_id != PLANE_CURSOR)
6443 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6444 else
6445 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6446
6447 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6448
6449 if (HAS_HW_SAGV_WM(dev_priv)) {
6450 if (plane_id != PLANE_CURSOR)
6451 val = intel_uncore_read(&dev_priv->uncore,
6452 PLANE_WM_SAGV(pipe, plane_id));
6453 else
6454 val = intel_uncore_read(&dev_priv->uncore,
6455 CUR_WM_SAGV(pipe));
6456
6457 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6458
6459 if (plane_id != PLANE_CURSOR)
6460 val = intel_uncore_read(&dev_priv->uncore,
6461 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6462 else
6463 val = intel_uncore_read(&dev_priv->uncore,
6464 CUR_WM_SAGV_TRANS(pipe));
6465
6466 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6467 } else if (DISPLAY_VER(dev_priv) >= 12) {
6468 wm->sagv.wm0 = wm->wm[0];
6469 wm->sagv.trans_wm = wm->trans_wm;
6470 }
6471 }
6472}
6473
6474void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6475{
6476 struct intel_dbuf_state *dbuf_state =
6477 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6478 struct intel_crtc *crtc;
6479
6480 if (IS_ALDERLAKE_P(dev_priv))
6481 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6482
6483 for_each_intel_crtc(&dev_priv->drm, crtc) {
6484 struct intel_crtc_state *crtc_state =
6485 to_intel_crtc_state(crtc->base.state);
6486 enum pipe pipe = crtc->pipe;
6487 unsigned int mbus_offset;
6488 enum plane_id plane_id;
6489
6490 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6491 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6492
6493 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6494
6495 for_each_plane_id_on_crtc(crtc, plane_id) {
6496 struct skl_ddb_entry *ddb_y =
6497 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6498 struct skl_ddb_entry *ddb_uv =
6499 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6500
6501 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6502 plane_id, ddb_y, ddb_uv);
6503
6504 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6505 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6506 }
6507
6508 dbuf_state->slices[pipe] =
6509 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6510
6511 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6512
6513 /*
6514 * Used for checking overlaps, so we need absolute
6515 * offsets instead of MBUS relative offsets.
6516 */
6517 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
6518 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6519 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6520
6521 drm_dbg_kms(&dev_priv->drm,
6522 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6523 crtc->base.base.id, crtc->base.name,
6524 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6525 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6526 yesno(dbuf_state->joined_mbus));
6527 }
6528
6529 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6530}
6531
6532static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6533{
6534 struct drm_device *dev = crtc->base.dev;
6535 struct drm_i915_private *dev_priv = to_i915(dev);
6536 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6537 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6538 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6539 enum pipe pipe = crtc->pipe;
6540
6541 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6542
6543 memset(active, 0, sizeof(*active));
6544
6545 active->pipe_enabled = crtc->active;
6546
6547 if (active->pipe_enabled) {
6548 u32 tmp = hw->wm_pipe[pipe];
6549
6550 /*
6551 * For active pipes LP0 watermark is marked as
6552 * enabled, and LP1+ watermaks as disabled since
6553 * we can't really reverse compute them in case
6554 * multiple pipes are active.
6555 */
6556 active->wm[0].enable = true;
6557 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6558 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6559 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6560 } else {
6561 int level, max_level = ilk_wm_max_level(dev_priv);
6562
6563 /*
6564 * For inactive pipes, all watermark levels
6565 * should be marked as enabled but zeroed,
6566 * which is what we'd compute them to.
6567 */
6568 for (level = 0; level <= max_level; level++)
6569 active->wm[level].enable = true;
6570 }
6571
6572 crtc->wm.active.ilk = *active;
6573}
6574
6575#define _FW_WM(value, plane) \
6576 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6577#define _FW_WM_VLV(value, plane) \
6578 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6579
6580static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6581 struct g4x_wm_values *wm)
6582{
6583 u32 tmp;
6584
6585 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6586 wm->sr.plane = _FW_WM(tmp, SR);
6587 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6588 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6589 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6590
6591 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6592 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6593 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6594 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6595 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6596 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6597 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6598
6599 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6600 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6601 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6602 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6603 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6604}
6605
6606static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6607 struct vlv_wm_values *wm)
6608{
6609 enum pipe pipe;
6610 u32 tmp;
6611
6612 for_each_pipe(dev_priv, pipe) {
6613 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6614
6615 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6616 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6617 wm->ddl[pipe].plane[PLANE_CURSOR] =
6618 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6619 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6620 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6621 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6622 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6623 }
6624
6625 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6626 wm->sr.plane = _FW_WM(tmp, SR);
6627 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6628 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6629 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6630
6631 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6632 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6633 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6634 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6635
6636 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6637 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6638
6639 if (IS_CHERRYVIEW(dev_priv)) {
6640 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6641 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6642 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6643
6644 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6645 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6646 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6647
6648 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6649 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6650 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6651
6652 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6653 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6654 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6655 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6656 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6657 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6658 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6659 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6660 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6661 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6662 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6663 } else {
6664 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6665 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6666 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6667
6668 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6669 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6670 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6671 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6672 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6673 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6674 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6675 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6676 }
6677}
6678
6679#undef _FW_WM
6680#undef _FW_WM_VLV
6681
6682void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6683{
6684 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6685 struct intel_crtc *crtc;
6686
6687 g4x_read_wm_values(dev_priv, wm);
6688
6689 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6690
6691 for_each_intel_crtc(&dev_priv->drm, crtc) {
6692 struct intel_crtc_state *crtc_state =
6693 to_intel_crtc_state(crtc->base.state);
6694 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6695 struct g4x_pipe_wm *raw;
6696 enum pipe pipe = crtc->pipe;
6697 enum plane_id plane_id;
6698 int level, max_level;
6699
6700 active->cxsr = wm->cxsr;
6701 active->hpll_en = wm->hpll_en;
6702 active->fbc_en = wm->fbc_en;
6703
6704 active->sr = wm->sr;
6705 active->hpll = wm->hpll;
6706
6707 for_each_plane_id_on_crtc(crtc, plane_id) {
6708 active->wm.plane[plane_id] =
6709 wm->pipe[pipe].plane[plane_id];
6710 }
6711
6712 if (wm->cxsr && wm->hpll_en)
6713 max_level = G4X_WM_LEVEL_HPLL;
6714 else if (wm->cxsr)
6715 max_level = G4X_WM_LEVEL_SR;
6716 else
6717 max_level = G4X_WM_LEVEL_NORMAL;
6718
6719 level = G4X_WM_LEVEL_NORMAL;
6720 raw = &crtc_state->wm.g4x.raw[level];
6721 for_each_plane_id_on_crtc(crtc, plane_id)
6722 raw->plane[plane_id] = active->wm.plane[plane_id];
6723
6724 if (++level > max_level)
6725 goto out;
6726
6727 raw = &crtc_state->wm.g4x.raw[level];
6728 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6729 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6730 raw->plane[PLANE_SPRITE0] = 0;
6731 raw->fbc = active->sr.fbc;
6732
6733 if (++level > max_level)
6734 goto out;
6735
6736 raw = &crtc_state->wm.g4x.raw[level];
6737 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6738 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6739 raw->plane[PLANE_SPRITE0] = 0;
6740 raw->fbc = active->hpll.fbc;
6741
6742 out:
6743 for_each_plane_id_on_crtc(crtc, plane_id)
6744 g4x_raw_plane_wm_set(crtc_state, level,
6745 plane_id, USHRT_MAX);
6746 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6747
6748 crtc_state->wm.g4x.optimal = *active;
6749 crtc_state->wm.g4x.intermediate = *active;
6750
6751 drm_dbg_kms(&dev_priv->drm,
6752 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6753 pipe_name(pipe),
6754 wm->pipe[pipe].plane[PLANE_PRIMARY],
6755 wm->pipe[pipe].plane[PLANE_CURSOR],
6756 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6757 }
6758
6759 drm_dbg_kms(&dev_priv->drm,
6760 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6761 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6762 drm_dbg_kms(&dev_priv->drm,
6763 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6764 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6765 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6766 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6767}
6768
6769void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6770{
6771 struct intel_plane *plane;
6772 struct intel_crtc *crtc;
6773
6774 mutex_lock(&dev_priv->wm.wm_mutex);
6775
6776 for_each_intel_plane(&dev_priv->drm, plane) {
6777 struct intel_crtc *crtc =
6778 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6779 struct intel_crtc_state *crtc_state =
6780 to_intel_crtc_state(crtc->base.state);
6781 struct intel_plane_state *plane_state =
6782 to_intel_plane_state(plane->base.state);
6783 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6784 enum plane_id plane_id = plane->id;
6785 int level;
6786
6787 if (plane_state->uapi.visible)
6788 continue;
6789
6790 for (level = 0; level < 3; level++) {
6791 struct g4x_pipe_wm *raw =
6792 &crtc_state->wm.g4x.raw[level];
6793
6794 raw->plane[plane_id] = 0;
6795 wm_state->wm.plane[plane_id] = 0;
6796 }
6797
6798 if (plane_id == PLANE_PRIMARY) {
6799 for (level = 0; level < 3; level++) {
6800 struct g4x_pipe_wm *raw =
6801 &crtc_state->wm.g4x.raw[level];
6802 raw->fbc = 0;
6803 }
6804
6805 wm_state->sr.fbc = 0;
6806 wm_state->hpll.fbc = 0;
6807 wm_state->fbc_en = false;
6808 }
6809 }
6810
6811 for_each_intel_crtc(&dev_priv->drm, crtc) {
6812 struct intel_crtc_state *crtc_state =
6813 to_intel_crtc_state(crtc->base.state);
6814
6815 crtc_state->wm.g4x.intermediate =
6816 crtc_state->wm.g4x.optimal;
6817 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6818 }
6819
6820 g4x_program_watermarks(dev_priv);
6821
6822 mutex_unlock(&dev_priv->wm.wm_mutex);
6823}
6824
6825void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6826{
6827 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6828 struct intel_crtc *crtc;
6829 u32 val;
6830
6831 vlv_read_wm_values(dev_priv, wm);
6832
6833 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6834 wm->level = VLV_WM_LEVEL_PM2;
6835
6836 if (IS_CHERRYVIEW(dev_priv)) {
6837 vlv_punit_get(dev_priv);
6838
6839 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6840 if (val & DSP_MAXFIFO_PM5_ENABLE)
6841 wm->level = VLV_WM_LEVEL_PM5;
6842
6843 /*
6844 * If DDR DVFS is disabled in the BIOS, Punit
6845 * will never ack the request. So if that happens
6846 * assume we don't have to enable/disable DDR DVFS
6847 * dynamically. To test that just set the REQ_ACK
6848 * bit to poke the Punit, but don't change the
6849 * HIGH/LOW bits so that we don't actually change
6850 * the current state.
6851 */
6852 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6853 val |= FORCE_DDR_FREQ_REQ_ACK;
6854 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6855
6856 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6857 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6858 drm_dbg_kms(&dev_priv->drm,
6859 "Punit not acking DDR DVFS request, "
6860 "assuming DDR DVFS is disabled\n");
6861 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6862 } else {
6863 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6864 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6865 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6866 }
6867
6868 vlv_punit_put(dev_priv);
6869 }
6870
6871 for_each_intel_crtc(&dev_priv->drm, crtc) {
6872 struct intel_crtc_state *crtc_state =
6873 to_intel_crtc_state(crtc->base.state);
6874 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6875 const struct vlv_fifo_state *fifo_state =
6876 &crtc_state->wm.vlv.fifo_state;
6877 enum pipe pipe = crtc->pipe;
6878 enum plane_id plane_id;
6879 int level;
6880
6881 vlv_get_fifo_size(crtc_state);
6882
6883 active->num_levels = wm->level + 1;
6884 active->cxsr = wm->cxsr;
6885
6886 for (level = 0; level < active->num_levels; level++) {
6887 struct g4x_pipe_wm *raw =
6888 &crtc_state->wm.vlv.raw[level];
6889
6890 active->sr[level].plane = wm->sr.plane;
6891 active->sr[level].cursor = wm->sr.cursor;
6892
6893 for_each_plane_id_on_crtc(crtc, plane_id) {
6894 active->wm[level].plane[plane_id] =
6895 wm->pipe[pipe].plane[plane_id];
6896
6897 raw->plane[plane_id] =
6898 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6899 fifo_state->plane[plane_id]);
6900 }
6901 }
6902
6903 for_each_plane_id_on_crtc(crtc, plane_id)
6904 vlv_raw_plane_wm_set(crtc_state, level,
6905 plane_id, USHRT_MAX);
6906 vlv_invalidate_wms(crtc, active, level);
6907
6908 crtc_state->wm.vlv.optimal = *active;
6909 crtc_state->wm.vlv.intermediate = *active;
6910
6911 drm_dbg_kms(&dev_priv->drm,
6912 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6913 pipe_name(pipe),
6914 wm->pipe[pipe].plane[PLANE_PRIMARY],
6915 wm->pipe[pipe].plane[PLANE_CURSOR],
6916 wm->pipe[pipe].plane[PLANE_SPRITE0],
6917 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6918 }
6919
6920 drm_dbg_kms(&dev_priv->drm,
6921 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6922 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6923}
6924
6925void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6926{
6927 struct intel_plane *plane;
6928 struct intel_crtc *crtc;
6929
6930 mutex_lock(&dev_priv->wm.wm_mutex);
6931
6932 for_each_intel_plane(&dev_priv->drm, plane) {
6933 struct intel_crtc *crtc =
6934 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6935 struct intel_crtc_state *crtc_state =
6936 to_intel_crtc_state(crtc->base.state);
6937 struct intel_plane_state *plane_state =
6938 to_intel_plane_state(plane->base.state);
6939 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6940 const struct vlv_fifo_state *fifo_state =
6941 &crtc_state->wm.vlv.fifo_state;
6942 enum plane_id plane_id = plane->id;
6943 int level;
6944
6945 if (plane_state->uapi.visible)
6946 continue;
6947
6948 for (level = 0; level < wm_state->num_levels; level++) {
6949 struct g4x_pipe_wm *raw =
6950 &crtc_state->wm.vlv.raw[level];
6951
6952 raw->plane[plane_id] = 0;
6953
6954 wm_state->wm[level].plane[plane_id] =
6955 vlv_invert_wm_value(raw->plane[plane_id],
6956 fifo_state->plane[plane_id]);
6957 }
6958 }
6959
6960 for_each_intel_crtc(&dev_priv->drm, crtc) {
6961 struct intel_crtc_state *crtc_state =
6962 to_intel_crtc_state(crtc->base.state);
6963
6964 crtc_state->wm.vlv.intermediate =
6965 crtc_state->wm.vlv.optimal;
6966 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6967 }
6968
6969 vlv_program_watermarks(dev_priv);
6970
6971 mutex_unlock(&dev_priv->wm.wm_mutex);
6972}
6973
6974/*
6975 * FIXME should probably kill this and improve
6976 * the real watermark readout/sanitation instead
6977 */
6978static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6979{
6980 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
6981 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
6982 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
6983
6984 /*
6985 * Don't touch WM1S_LP_EN here.
6986 * Doing so could cause underruns.
6987 */
6988}
6989
6990void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6991{
6992 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6993 struct intel_crtc *crtc;
6994
6995 ilk_init_lp_watermarks(dev_priv);
6996
6997 for_each_intel_crtc(&dev_priv->drm, crtc)
6998 ilk_pipe_wm_get_hw_state(crtc);
6999
7000 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7001 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7002 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7003
7004 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7005 if (DISPLAY_VER(dev_priv) >= 7) {
7006 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7007 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7008 }
7009
7010 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7011 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7012 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7013 else if (IS_IVYBRIDGE(dev_priv))
7014 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7015 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7016
7017 hw->enable_fbc_wm =
7018 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7019}
7020
7021/**
7022 * intel_update_watermarks - update FIFO watermark values based on current modes
7023 * @crtc: the #intel_crtc on which to compute the WM
7024 *
7025 * Calculate watermark values for the various WM regs based on current mode
7026 * and plane configuration.
7027 *
7028 * There are several cases to deal with here:
7029 * - normal (i.e. non-self-refresh)
7030 * - self-refresh (SR) mode
7031 * - lines are large relative to FIFO size (buffer can hold up to 2)
7032 * - lines are small relative to FIFO size (buffer can hold more than 2
7033 * lines), so need to account for TLB latency
7034 *
7035 * The normal calculation is:
7036 * watermark = dotclock * bytes per pixel * latency
7037 * where latency is platform & configuration dependent (we assume pessimal
7038 * values here).
7039 *
7040 * The SR calculation is:
7041 * watermark = (trunc(latency/line time)+1) * surface width *
7042 * bytes per pixel
7043 * where
7044 * line time = htotal / dotclock
7045 * surface width = hdisplay for normal plane and 64 for cursor
7046 * and latency is assumed to be high, as above.
7047 *
7048 * The final value programmed to the register should always be rounded up,
7049 * and include an extra 2 entries to account for clock crossings.
7050 *
7051 * We don't use the sprite, so we can ignore that. And on Crestline we have
7052 * to set the non-SR watermarks to 8.
7053 */
7054void intel_update_watermarks(struct intel_crtc *crtc)
7055{
7056 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7057
7058 if (dev_priv->display.update_wm)
7059 dev_priv->display.update_wm(crtc);
7060}
7061
7062void intel_enable_ipc(struct drm_i915_private *dev_priv)
7063{
7064 u32 val;
7065
7066 if (!HAS_IPC(dev_priv))
7067 return;
7068
7069 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7070
7071 if (dev_priv->ipc_enabled)
7072 val |= DISP_IPC_ENABLE;
7073 else
7074 val &= ~DISP_IPC_ENABLE;
7075
7076 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7077}
7078
7079static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7080{
7081 /* Display WA #0477 WaDisableIPC: skl */
7082 if (IS_SKYLAKE(dev_priv))
7083 return false;
7084
7085 /* Display WA #1141: SKL:all KBL:all CFL */
7086 if (IS_KABYLAKE(dev_priv) ||
7087 IS_COFFEELAKE(dev_priv) ||
7088 IS_COMETLAKE(dev_priv))
7089 return dev_priv->dram_info.symmetric_memory;
7090
7091 return true;
7092}
7093
7094void intel_init_ipc(struct drm_i915_private *dev_priv)
7095{
7096 if (!HAS_IPC(dev_priv))
7097 return;
7098
7099 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7100
7101 intel_enable_ipc(dev_priv);
7102}
7103
7104static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7105{
7106 /*
7107 * On Ibex Peak and Cougar Point, we need to disable clock
7108 * gating for the panel power sequencer or it will fail to
7109 * start up when no ports are active.
7110 */
7111 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7112}
7113
7114static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7115{
7116 enum pipe pipe;
7117
7118 for_each_pipe(dev_priv, pipe) {
7119 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7120 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7121 DISPPLANE_TRICKLE_FEED_DISABLE);
7122
7123 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7124 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7125 }
7126}
7127
7128static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7129{
7130 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7131
7132 /*
7133 * Required for FBC
7134 * WaFbcDisableDpfcClockGating:ilk
7135 */
7136 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7137 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7138 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7139
7140 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7141 MARIUNIT_CLOCK_GATE_DISABLE |
7142 SVSMUNIT_CLOCK_GATE_DISABLE);
7143 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7144 VFMUNIT_CLOCK_GATE_DISABLE);
7145
7146 /*
7147 * According to the spec the following bits should be set in
7148 * order to enable memory self-refresh
7149 * The bit 22/21 of 0x42004
7150 * The bit 5 of 0x42020
7151 * The bit 15 of 0x45000
7152 */
7153 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7154 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7155 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7156 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7157 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7158 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7159 DISP_FBC_WM_DIS));
7160
7161 /*
7162 * Based on the document from hardware guys the following bits
7163 * should be set unconditionally in order to enable FBC.
7164 * The bit 22 of 0x42000
7165 * The bit 22 of 0x42004
7166 * The bit 7,8,9 of 0x42020.
7167 */
7168 if (IS_IRONLAKE_M(dev_priv)) {
7169 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7170 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7171 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7172 ILK_FBCQ_DIS);
7173 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7174 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7175 ILK_DPARB_GATE);
7176 }
7177
7178 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7179
7180 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7181 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7182 ILK_ELPIN_409_SELECT);
7183
7184 g4x_disable_trickle_feed(dev_priv);
7185
7186 ibx_init_clock_gating(dev_priv);
7187}
7188
7189static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7190{
7191 enum pipe pipe;
7192 u32 val;
7193
7194 /*
7195 * On Ibex Peak and Cougar Point, we need to disable clock
7196 * gating for the panel power sequencer or it will fail to
7197 * start up when no ports are active.
7198 */
7199 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7200 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7201 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7202 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7203 DPLS_EDP_PPS_FIX_DIS);
7204 /* The below fixes the weird display corruption, a few pixels shifted
7205 * downward, on (only) LVDS of some HP laptops with IVY.
7206 */
7207 for_each_pipe(dev_priv, pipe) {
7208 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7209 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7210 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7211 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7212 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7213 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7214 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7215 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7216 }
7217 /* WADP0ClockGatingDisable */
7218 for_each_pipe(dev_priv, pipe) {
7219 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7220 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7221 }
7222}
7223
7224static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7225{
7226 u32 tmp;
7227
7228 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7229 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7230 drm_dbg_kms(&dev_priv->drm,
7231 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7232 tmp);
7233}
7234
7235static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7236{
7237 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7238
7239 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7240
7241 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7242 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7243 ILK_ELPIN_409_SELECT);
7244
7245 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7246 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7247 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7248 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7249
7250 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7251 * gating disable must be set. Failure to set it results in
7252 * flickering pixels due to Z write ordering failures after
7253 * some amount of runtime in the Mesa "fire" demo, and Unigine
7254 * Sanctuary and Tropics, and apparently anything else with
7255 * alpha test or pixel discard.
7256 *
7257 * According to the spec, bit 11 (RCCUNIT) must also be set,
7258 * but we didn't debug actual testcases to find it out.
7259 *
7260 * WaDisableRCCUnitClockGating:snb
7261 * WaDisableRCPBUnitClockGating:snb
7262 */
7263 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7264 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7265 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7266
7267 /*
7268 * According to the spec the following bits should be
7269 * set in order to enable memory self-refresh and fbc:
7270 * The bit21 and bit22 of 0x42000
7271 * The bit21 and bit22 of 0x42004
7272 * The bit5 and bit7 of 0x42020
7273 * The bit14 of 0x70180
7274 * The bit14 of 0x71180
7275 *
7276 * WaFbcAsynchFlipDisableFbcQueue:snb
7277 */
7278 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7279 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7280 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7281 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7282 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7283 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7284 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7285 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7286 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7287 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7288
7289 g4x_disable_trickle_feed(dev_priv);
7290
7291 cpt_init_clock_gating(dev_priv);
7292
7293 gen6_check_mch_setup(dev_priv);
7294}
7295
7296static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7297{
7298 /*
7299 * TODO: this bit should only be enabled when really needed, then
7300 * disabled when not needed anymore in order to save power.
7301 */
7302 if (HAS_PCH_LPT_LP(dev_priv))
7303 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7304 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7305 PCH_LP_PARTITION_LEVEL_DISABLE);
7306
7307 /* WADPOClockGatingDisable:hsw */
7308 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7309 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7310 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7311}
7312
7313static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7314{
7315 if (HAS_PCH_LPT_LP(dev_priv)) {
7316 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7317
7318 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7319 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7320 }
7321}
7322
7323static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7324 int general_prio_credits,
7325 int high_prio_credits)
7326{
7327 u32 misccpctl;
7328 u32 val;
7329
7330 /* WaTempDisableDOPClkGating:bdw */
7331 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7332 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7333
7334 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7335 val &= ~L3_PRIO_CREDITS_MASK;
7336 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7337 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7338 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7339
7340 /*
7341 * Wait at least 100 clocks before re-enabling clock gating.
7342 * See the definition of L3SQCREG1 in BSpec.
7343 */
7344 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7345 udelay(1);
7346 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7347}
7348
7349static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7350{
7351 /* Wa_1409120013:icl,ehl */
7352 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7353 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7354
7355 /* This is not an Wa. Enable to reduce Sampler power */
7356 intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7357 intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7358
7359 /*Wa_14010594013:icl, ehl */
7360 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7361 0, CNL_DELAY_PMRSP);
7362}
7363
7364static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7365{
7366 /* Wa_1409120013:tgl,rkl,adl_s,dg1 */
7367 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7368 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7369
7370 /* Wa_1409825376:tgl (pre-prod)*/
7371 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1))
7372 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7373 TGL_VRH_GATING_DIS);
7374
7375 /* Wa_14011059788:tgl,rkl,adl_s,dg1 */
7376 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7377 0, DFR_DISABLE);
7378
7379 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7380 if (DISPLAY_VER(dev_priv) == 12)
7381 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7382 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7383}
7384
7385static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7386{
7387 gen12lp_init_clock_gating(dev_priv);
7388
7389 /* Wa_22011091694:adlp */
7390 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7391}
7392
7393static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7394{
7395 gen12lp_init_clock_gating(dev_priv);
7396
7397 /* Wa_1409836686:dg1[a0] */
7398 if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0))
7399 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7400 DPT_GATING_DIS);
7401}
7402
7403static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7404{
7405 if (!HAS_PCH_CNP(dev_priv))
7406 return;
7407
7408 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7409 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7410 CNP_PWM_CGE_GATING_DISABLE);
7411}
7412
7413static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7414{
7415 u32 val;
7416 cnp_init_clock_gating(dev_priv);
7417
7418 /* This is not an Wa. Enable for better image quality */
7419 intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3,
7420 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7421
7422 /* WaEnableChickenDCPR:cnl */
7423 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7424 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7425
7426 /*
7427 * WaFbcWakeMemOn:cnl
7428 * Display WA #0859: cnl
7429 */
7430 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7431 DISP_FBC_MEMORY_WAKE);
7432
7433 val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE);
7434 /* ReadHitWriteOnlyDisable:cnl */
7435 val |= RCCUNIT_CLKGATE_DIS;
7436 intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val);
7437
7438 /* Wa_2201832410:cnl */
7439 val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE);
7440 val |= GWUNIT_CLKGATE_DIS;
7441 intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7442
7443 /* WaDisableVFclkgate:cnl */
7444 /* WaVFUnitClockGatingDisable:cnl */
7445 val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE);
7446 val |= VFUNIT_CLKGATE_DIS;
7447 intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val);
7448}
7449
7450static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7451{
7452 cnp_init_clock_gating(dev_priv);
7453 gen9_init_clock_gating(dev_priv);
7454
7455 /* WAC6entrylatency:cfl */
7456 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7457 FBC_LLC_FULLY_OPEN);
7458
7459 /*
7460 * WaFbcTurnOffFbcWatermark:cfl
7461 * Display WA #0562: cfl
7462 */
7463 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7464 DISP_FBC_WM_DIS);
7465
7466 /*
7467 * WaFbcNukeOnHostModify:cfl
7468 * Display WA #0873: cfl
7469 */
7470 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7471 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7472}
7473
7474static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7475{
7476 gen9_init_clock_gating(dev_priv);
7477
7478 /* WAC6entrylatency:kbl */
7479 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7480 FBC_LLC_FULLY_OPEN);
7481
7482 /* WaDisableSDEUnitClockGating:kbl */
7483 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7484 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7485 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7486
7487 /* WaDisableGamClockGating:kbl */
7488 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7489 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7490 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7491
7492 /*
7493 * WaFbcTurnOffFbcWatermark:kbl
7494 * Display WA #0562: kbl
7495 */
7496 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7497 DISP_FBC_WM_DIS);
7498
7499 /*
7500 * WaFbcNukeOnHostModify:kbl
7501 * Display WA #0873: kbl
7502 */
7503 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7504 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7505}
7506
7507static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7508{
7509 gen9_init_clock_gating(dev_priv);
7510
7511 /* WaDisableDopClockGating:skl */
7512 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7513 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7514
7515 /* WAC6entrylatency:skl */
7516 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7517 FBC_LLC_FULLY_OPEN);
7518
7519 /*
7520 * WaFbcTurnOffFbcWatermark:skl
7521 * Display WA #0562: skl
7522 */
7523 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7524 DISP_FBC_WM_DIS);
7525
7526 /*
7527 * WaFbcNukeOnHostModify:skl
7528 * Display WA #0873: skl
7529 */
7530 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7531 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7532
7533 /*
7534 * WaFbcHighMemBwCorruptionAvoidance:skl
7535 * Display WA #0883: skl
7536 */
7537 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7538 ILK_DPFC_DISABLE_DUMMY0);
7539}
7540
7541static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7542{
7543 enum pipe pipe;
7544
7545 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7546 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7547 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7548 HSW_FBCQ_DIS);
7549
7550 /* WaSwitchSolVfFArbitrationPriority:bdw */
7551 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7552
7553 /* WaPsrDPAMaskVBlankInSRD:bdw */
7554 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7555 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7556
7557 for_each_pipe(dev_priv, pipe) {
7558 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7559 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7560 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7561 BDW_DPRS_MASK_VBLANK_SRD);
7562
7563 /* Undocumented but fixes async flip + VT-d corruption */
7564 if (intel_vtd_active())
7565 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7566 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7567 }
7568
7569 /* WaVSRefCountFullforceMissDisable:bdw */
7570 /* WaDSRefCountFullforceMissDisable:bdw */
7571 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7572 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7573 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7574
7575 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7576 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7577
7578 /* WaDisableSDEUnitClockGating:bdw */
7579 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7580 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7581
7582 /* WaProgramL3SqcReg1Default:bdw */
7583 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7584
7585 /* WaKVMNotificationOnConfigChange:bdw */
7586 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7587 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7588
7589 lpt_init_clock_gating(dev_priv);
7590
7591 /* WaDisableDopClockGating:bdw
7592 *
7593 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7594 * clock gating.
7595 */
7596 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7597 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7598}
7599
7600static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7601{
7602 enum pipe pipe;
7603
7604 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7605 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7606 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7607 HSW_FBCQ_DIS);
7608
7609 for_each_pipe(dev_priv, pipe) {
7610 /* Undocumented but fixes async flip + VT-d corruption */
7611 if (intel_vtd_active())
7612 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7613 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7614 }
7615
7616 /* This is required by WaCatErrorRejectionIssue:hsw */
7617 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7618 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7619 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7620
7621 /* WaSwitchSolVfFArbitrationPriority:hsw */
7622 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7623
7624 lpt_init_clock_gating(dev_priv);
7625}
7626
7627static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7628{
7629 u32 snpcr;
7630
7631 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7632
7633 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7634 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7635 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7636 ILK_FBCQ_DIS);
7637
7638 /* WaDisableBackToBackFlipFix:ivb */
7639 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7640 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7641 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7642
7643 if (IS_IVB_GT1(dev_priv))
7644 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7645 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7646 else {
7647 /* must write both registers */
7648 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7649 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7650 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7651 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7652 }
7653
7654 /*
7655 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7656 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7657 */
7658 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7659 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7660
7661 /* This is required by WaCatErrorRejectionIssue:ivb */
7662 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7663 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7664 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7665
7666 g4x_disable_trickle_feed(dev_priv);
7667
7668 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7669 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7670 snpcr |= GEN6_MBC_SNPCR_MED;
7671 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7672
7673 if (!HAS_PCH_NOP(dev_priv))
7674 cpt_init_clock_gating(dev_priv);
7675
7676 gen6_check_mch_setup(dev_priv);
7677}
7678
7679static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7680{
7681 /* WaDisableBackToBackFlipFix:vlv */
7682 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7683 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7684 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7685
7686 /* WaDisableDopClockGating:vlv */
7687 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7688 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7689
7690 /* This is required by WaCatErrorRejectionIssue:vlv */
7691 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7692 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7693 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7694
7695 /*
7696 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7697 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7698 */
7699 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7700 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7701
7702 /* WaDisableL3Bank2xClockGate:vlv
7703 * Disabling L3 clock gating- MMIO 940c[25] = 1
7704 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7705 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7706 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7707
7708 /*
7709 * WaDisableVLVClockGating_VBIIssue:vlv
7710 * Disable clock gating on th GCFG unit to prevent a delay
7711 * in the reporting of vblank events.
7712 */
7713 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7714}
7715
7716static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7717{
7718 /* WaVSRefCountFullforceMissDisable:chv */
7719 /* WaDSRefCountFullforceMissDisable:chv */
7720 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7721 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7722 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7723
7724 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7725 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7726 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7727
7728 /* WaDisableCSUnitClockGating:chv */
7729 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7730 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7731
7732 /* WaDisableSDEUnitClockGating:chv */
7733 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7734 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7735
7736 /*
7737 * WaProgramL3SqcReg1Default:chv
7738 * See gfxspecs/Related Documents/Performance Guide/
7739 * LSQC Setting Recommendations.
7740 */
7741 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7742}
7743
7744static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7745{
7746 u32 dspclk_gate;
7747
7748 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7749 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7750 GS_UNIT_CLOCK_GATE_DISABLE |
7751 CL_UNIT_CLOCK_GATE_DISABLE);
7752 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7753 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7754 OVRUNIT_CLOCK_GATE_DISABLE |
7755 OVCUNIT_CLOCK_GATE_DISABLE;
7756 if (IS_GM45(dev_priv))
7757 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7758 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7759
7760 g4x_disable_trickle_feed(dev_priv);
7761}
7762
7763static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7764{
7765 struct intel_uncore *uncore = &dev_priv->uncore;
7766
7767 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7768 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7769 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7770 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7771 intel_uncore_write16(uncore, DEUC, 0);
7772 intel_uncore_write(uncore,
7773 MI_ARB_STATE,
7774 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7775}
7776
7777static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7778{
7779 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7780 I965_RCC_CLOCK_GATE_DISABLE |
7781 I965_RCPB_CLOCK_GATE_DISABLE |
7782 I965_ISC_CLOCK_GATE_DISABLE |
7783 I965_FBC_CLOCK_GATE_DISABLE);
7784 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7785 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7786 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7787}
7788
7789static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7790{
7791 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7792
7793 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7794 DSTATE_DOT_CLOCK_GATING;
7795 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7796
7797 if (IS_PINEVIEW(dev_priv))
7798 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7799
7800 /* IIR "flip pending" means done if this bit is set */
7801 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7802
7803 /* interrupts should cause a wake up from C3 */
7804 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7805
7806 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7807 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7808
7809 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7810 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7811}
7812
7813static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7814{
7815 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7816
7817 /* interrupts should cause a wake up from C3 */
7818 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7819 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7820
7821 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7822 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7823
7824 /*
7825 * Have FBC ignore 3D activity since we use software
7826 * render tracking, and otherwise a pure 3D workload
7827 * (even if it just renders a single frame and then does
7828 * abosultely nothing) would not allow FBC to recompress
7829 * until a 2D blit occurs.
7830 */
7831 intel_uncore_write(&dev_priv->uncore, SCPD0,
7832 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7833}
7834
7835static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7836{
7837 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7838 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7839 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7840}
7841
7842void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7843{
7844 dev_priv->display.init_clock_gating(dev_priv);
7845}
7846
7847void intel_suspend_hw(struct drm_i915_private *dev_priv)
7848{
7849 if (HAS_PCH_LPT(dev_priv))
7850 lpt_suspend_hw(dev_priv);
7851}
7852
7853static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7854{
7855 drm_dbg_kms(&dev_priv->drm,
7856 "No clock gating settings or workarounds applied.\n");
7857}
7858
7859/**
7860 * intel_init_clock_gating_hooks - setup the clock gating hooks
7861 * @dev_priv: device private
7862 *
7863 * Setup the hooks that configure which clocks of a given platform can be
7864 * gated and also apply various GT and display specific workarounds for these
7865 * platforms. Note that some GT specific workarounds are applied separately
7866 * when GPU contexts or batchbuffers start their execution.
7867 */
7868void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7869{
7870 if (IS_ALDERLAKE_P(dev_priv))
7871 dev_priv->display.init_clock_gating = adlp_init_clock_gating;
7872 else if (IS_DG1(dev_priv))
7873 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7874 else if (GRAPHICS_VER(dev_priv) == 12)
7875 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7876 else if (GRAPHICS_VER(dev_priv) == 11)
7877 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7878 else if (IS_CANNONLAKE(dev_priv))
7879 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7880 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7881 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7882 else if (IS_SKYLAKE(dev_priv))
7883 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7884 else if (IS_KABYLAKE(dev_priv))
7885 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7886 else if (IS_BROXTON(dev_priv))
7887 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7888 else if (IS_GEMINILAKE(dev_priv))
7889 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7890 else if (IS_BROADWELL(dev_priv))
7891 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7892 else if (IS_CHERRYVIEW(dev_priv))
7893 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7894 else if (IS_HASWELL(dev_priv))
7895 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7896 else if (IS_IVYBRIDGE(dev_priv))
7897 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7898 else if (IS_VALLEYVIEW(dev_priv))
7899 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7900 else if (GRAPHICS_VER(dev_priv) == 6)
7901 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7902 else if (GRAPHICS_VER(dev_priv) == 5)
7903 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7904 else if (IS_G4X(dev_priv))
7905 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7906 else if (IS_I965GM(dev_priv))
7907 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7908 else if (IS_I965G(dev_priv))
7909 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7910 else if (GRAPHICS_VER(dev_priv) == 3)
7911 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7912 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7913 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7914 else if (GRAPHICS_VER(dev_priv) == 2)
7915 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7916 else {
7917 MISSING_CASE(INTEL_DEVID(dev_priv));
7918 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7919 }
7920}
7921
7922/* Set up chip specific power management-related functions */
7923void intel_init_pm(struct drm_i915_private *dev_priv)
7924{
7925 /* For cxsr */
7926 if (IS_PINEVIEW(dev_priv))
7927 pnv_get_mem_freq(dev_priv);
7928 else if (GRAPHICS_VER(dev_priv) == 5)
7929 ilk_get_mem_freq(dev_priv);
7930
7931 if (intel_has_sagv(dev_priv))
7932 skl_setup_sagv_block_time(dev_priv);
7933
7934 /* For FIFO watermark updates */
7935 if (DISPLAY_VER(dev_priv) >= 9) {
7936 skl_setup_wm_latency(dev_priv);
7937 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7938 } else if (HAS_PCH_SPLIT(dev_priv)) {
7939 ilk_setup_wm_latency(dev_priv);
7940
7941 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
7942 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7943 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
7944 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7945 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7946 dev_priv->display.compute_intermediate_wm =
7947 ilk_compute_intermediate_wm;
7948 dev_priv->display.initial_watermarks =
7949 ilk_initial_watermarks;
7950 dev_priv->display.optimize_watermarks =
7951 ilk_optimize_watermarks;
7952 } else {
7953 drm_dbg_kms(&dev_priv->drm,
7954 "Failed to read display plane latency. "
7955 "Disable CxSR\n");
7956 }
7957 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7958 vlv_setup_wm_latency(dev_priv);
7959 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7960 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7961 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7962 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7963 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7964 } else if (IS_G4X(dev_priv)) {
7965 g4x_setup_wm_latency(dev_priv);
7966 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7967 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7968 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7969 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7970 } else if (IS_PINEVIEW(dev_priv)) {
7971 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7972 dev_priv->is_ddr3,
7973 dev_priv->fsb_freq,
7974 dev_priv->mem_freq)) {
7975 drm_info(&dev_priv->drm,
7976 "failed to find known CxSR latency "
7977 "(found ddr%s fsb freq %d, mem freq %d), "
7978 "disabling CxSR\n",
7979 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7980 dev_priv->fsb_freq, dev_priv->mem_freq);
7981 /* Disable CxSR and never update its watermark again */
7982 intel_set_memory_cxsr(dev_priv, false);
7983 dev_priv->display.update_wm = NULL;
7984 } else
7985 dev_priv->display.update_wm = pnv_update_wm;
7986 } else if (DISPLAY_VER(dev_priv) == 4) {
7987 dev_priv->display.update_wm = i965_update_wm;
7988 } else if (DISPLAY_VER(dev_priv) == 3) {
7989 dev_priv->display.update_wm = i9xx_update_wm;
7990 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7991 } else if (DISPLAY_VER(dev_priv) == 2) {
7992 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7993 dev_priv->display.update_wm = i845_update_wm;
7994 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7995 } else {
7996 dev_priv->display.update_wm = i9xx_update_wm;
7997 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7998 }
7999 } else {
8000 drm_err(&dev_priv->drm,
8001 "unexpected fall-through in %s\n", __func__);
8002 }
8003}
8004
8005void intel_pm_setup(struct drm_i915_private *dev_priv)
8006{
8007 dev_priv->runtime_pm.suspended = false;
8008 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8009}
8010
8011static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8012{
8013 struct intel_dbuf_state *dbuf_state;
8014
8015 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8016 if (!dbuf_state)
8017 return NULL;
8018
8019 return &dbuf_state->base;
8020}
8021
8022static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8023 struct intel_global_state *state)
8024{
8025 kfree(state);
8026}
8027
8028static const struct intel_global_state_funcs intel_dbuf_funcs = {
8029 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8030 .atomic_destroy_state = intel_dbuf_destroy_state,
8031};
8032
8033struct intel_dbuf_state *
8034intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8035{
8036 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8037 struct intel_global_state *dbuf_state;
8038
8039 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8040 if (IS_ERR(dbuf_state))
8041 return ERR_CAST(dbuf_state);
8042
8043 return to_intel_dbuf_state(dbuf_state);
8044}
8045
8046int intel_dbuf_init(struct drm_i915_private *dev_priv)
8047{
8048 struct intel_dbuf_state *dbuf_state;
8049
8050 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8051 if (!dbuf_state)
8052 return -ENOMEM;
8053
8054 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8055 &dbuf_state->base, &intel_dbuf_funcs);
8056
8057 return 0;
8058}
8059
8060/*
8061 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8062 * update the request state of all DBUS slices.
8063 */
8064static void update_mbus_pre_enable(struct intel_atomic_state *state)
8065{
8066 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8067 u32 mbus_ctl, dbuf_min_tracker_val;
8068 enum dbuf_slice slice;
8069 const struct intel_dbuf_state *dbuf_state =
8070 intel_atomic_get_new_dbuf_state(state);
8071
8072 if (!IS_ALDERLAKE_P(dev_priv))
8073 return;
8074
8075 /*
8076 * TODO: Implement vblank synchronized MBUS joining changes.
8077 * Must be properly coordinated with dbuf reprogramming.
8078 */
8079 if (dbuf_state->joined_mbus) {
8080 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8081 MBUS_JOIN_PIPE_SELECT_NONE;
8082 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8083 } else {
8084 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8085 MBUS_JOIN_PIPE_SELECT_NONE;
8086 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8087 }
8088
8089 intel_de_rmw(dev_priv, MBUS_CTL,
8090 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8091 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8092
8093 for_each_dbuf_slice(dev_priv, slice)
8094 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8095 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8096 dbuf_min_tracker_val);
8097}
8098
8099void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8100{
8101 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8102 const struct intel_dbuf_state *new_dbuf_state =
8103 intel_atomic_get_new_dbuf_state(state);
8104 const struct intel_dbuf_state *old_dbuf_state =
8105 intel_atomic_get_old_dbuf_state(state);
8106
8107 if (!new_dbuf_state ||
8108 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8109 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8110 return;
8111
8112 WARN_ON(!new_dbuf_state->base.changed);
8113
8114 update_mbus_pre_enable(state);
8115 gen9_dbuf_slices_update(dev_priv,
8116 old_dbuf_state->enabled_slices |
8117 new_dbuf_state->enabled_slices);
8118}
8119
8120void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8121{
8122 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8123 const struct intel_dbuf_state *new_dbuf_state =
8124 intel_atomic_get_new_dbuf_state(state);
8125 const struct intel_dbuf_state *old_dbuf_state =
8126 intel_atomic_get_old_dbuf_state(state);
8127
8128 if (!new_dbuf_state ||
8129 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8130 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8131 return;
8132
8133 WARN_ON(!new_dbuf_state->base.changed);
8134
8135 gen9_dbuf_slices_update(dev_priv,
8136 new_dbuf_state->enabled_slices);
8137}