Loading...
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include <linux/module.h>
29#include <linux/pm_runtime.h>
30
31#include <drm/drm_atomic_helper.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_plane_helper.h>
34
35#include "display/intel_atomic.h"
36#include "display/intel_atomic_plane.h"
37#include "display/intel_bw.h"
38#include "display/intel_de.h"
39#include "display/intel_display_types.h"
40#include "display/intel_fbc.h"
41#include "display/intel_sprite.h"
42#include "display/skl_universal_plane.h"
43
44#include "gt/intel_llc.h"
45
46#include "i915_drv.h"
47#include "i915_fixed.h"
48#include "i915_irq.h"
49#include "i915_trace.h"
50#include "intel_pm.h"
51#include "intel_sideband.h"
52#include "../../../platform/x86/intel_ips.h"
53
54/* Stores plane specific WM parameters */
55struct skl_wm_params {
56 bool x_tiled, y_tiled;
57 bool rc_surface;
58 bool is_planar;
59 u32 width;
60 u8 cpp;
61 u32 plane_pixel_rate;
62 u32 y_min_scanlines;
63 u32 plane_bytes_per_line;
64 uint_fixed_16_16_t plane_blocks_per_line;
65 uint_fixed_16_16_t y_tile_minimum;
66 u32 linetime_us;
67 u32 dbuf_block_size;
68};
69
70/* used in computing the new watermarks state */
71struct intel_wm_config {
72 unsigned int num_pipes_active;
73 bool sprites_enabled;
74 bool sprites_scaled;
75};
76
77static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
78{
79 enum pipe pipe;
80
81 if (HAS_LLC(dev_priv)) {
82 /*
83 * WaCompressedResourceDisplayNewHashMode:skl,kbl
84 * Display WA #0390: skl,kbl
85 *
86 * Must match Sampler, Pixel Back End, and Media. See
87 * WaCompressedResourceSamplerPbeMediaNewHashMode.
88 */
89 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
90 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
91 SKL_DE_COMPRESSED_HASH_MODE);
92 }
93
94 for_each_pipe(dev_priv, pipe) {
95 /*
96 * "Plane N strech max must be programmed to 11b (x1)
97 * when Async flips are enabled on that plane."
98 */
99 if (!IS_GEMINILAKE(dev_priv) && intel_vtd_active())
100 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
101 SKL_PLANE1_STRETCH_MAX_MASK, SKL_PLANE1_STRETCH_MAX_X1);
102 }
103
104 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
105 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
106 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
107
108 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
109 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
110 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
111
112 /*
113 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
114 * Display WA #0859: skl,bxt,kbl,glk,cfl
115 */
116 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
117 DISP_FBC_MEMORY_WAKE);
118}
119
120static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
121{
122 gen9_init_clock_gating(dev_priv);
123
124 /* WaDisableSDEUnitClockGating:bxt */
125 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
126 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
127
128 /*
129 * FIXME:
130 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
131 */
132 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
133 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
134
135 /*
136 * Wa: Backlight PWM may stop in the asserted state, causing backlight
137 * to stay fully on.
138 */
139 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
140 PWM1_GATING_DIS | PWM2_GATING_DIS);
141
142 /*
143 * Lower the display internal timeout.
144 * This is needed to avoid any hard hangs when DSI port PLL
145 * is off and a MMIO access is attempted by any privilege
146 * application, using batch buffers or any other means.
147 */
148 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
149
150 /*
151 * WaFbcTurnOffFbcWatermark:bxt
152 * Display WA #0562: bxt
153 */
154 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
155 DISP_FBC_WM_DIS);
156
157 /*
158 * WaFbcHighMemBwCorruptionAvoidance:bxt
159 * Display WA #0883: bxt
160 */
161 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
162 ILK_DPFC_DISABLE_DUMMY0);
163}
164
165static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
166{
167 gen9_init_clock_gating(dev_priv);
168
169 /*
170 * WaDisablePWMClockGating:glk
171 * Backlight PWM may stop in the asserted state, causing backlight
172 * to stay fully on.
173 */
174 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
175 PWM1_GATING_DIS | PWM2_GATING_DIS);
176}
177
178static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
179{
180 u32 tmp;
181
182 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
183
184 switch (tmp & CLKCFG_FSB_MASK) {
185 case CLKCFG_FSB_533:
186 dev_priv->fsb_freq = 533; /* 133*4 */
187 break;
188 case CLKCFG_FSB_800:
189 dev_priv->fsb_freq = 800; /* 200*4 */
190 break;
191 case CLKCFG_FSB_667:
192 dev_priv->fsb_freq = 667; /* 167*4 */
193 break;
194 case CLKCFG_FSB_400:
195 dev_priv->fsb_freq = 400; /* 100*4 */
196 break;
197 }
198
199 switch (tmp & CLKCFG_MEM_MASK) {
200 case CLKCFG_MEM_533:
201 dev_priv->mem_freq = 533;
202 break;
203 case CLKCFG_MEM_667:
204 dev_priv->mem_freq = 667;
205 break;
206 case CLKCFG_MEM_800:
207 dev_priv->mem_freq = 800;
208 break;
209 }
210
211 /* detect pineview DDR3 setting */
212 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
213 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
214}
215
216static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
217{
218 u16 ddrpll, csipll;
219
220 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
221 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
222
223 switch (ddrpll & 0xff) {
224 case 0xc:
225 dev_priv->mem_freq = 800;
226 break;
227 case 0x10:
228 dev_priv->mem_freq = 1066;
229 break;
230 case 0x14:
231 dev_priv->mem_freq = 1333;
232 break;
233 case 0x18:
234 dev_priv->mem_freq = 1600;
235 break;
236 default:
237 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
238 ddrpll & 0xff);
239 dev_priv->mem_freq = 0;
240 break;
241 }
242
243 switch (csipll & 0x3ff) {
244 case 0x00c:
245 dev_priv->fsb_freq = 3200;
246 break;
247 case 0x00e:
248 dev_priv->fsb_freq = 3733;
249 break;
250 case 0x010:
251 dev_priv->fsb_freq = 4266;
252 break;
253 case 0x012:
254 dev_priv->fsb_freq = 4800;
255 break;
256 case 0x014:
257 dev_priv->fsb_freq = 5333;
258 break;
259 case 0x016:
260 dev_priv->fsb_freq = 5866;
261 break;
262 case 0x018:
263 dev_priv->fsb_freq = 6400;
264 break;
265 default:
266 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
267 csipll & 0x3ff);
268 dev_priv->fsb_freq = 0;
269 break;
270 }
271}
272
273static const struct cxsr_latency cxsr_latency_table[] = {
274 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
275 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
276 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
277 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
278 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
279
280 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
281 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
282 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
283 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
284 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
285
286 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
287 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
288 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
289 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
290 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
291
292 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
293 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
294 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
295 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
296 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
297
298 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
299 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
300 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
301 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
302 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
303
304 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
305 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
306 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
307 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
308 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
309};
310
311static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
312 bool is_ddr3,
313 int fsb,
314 int mem)
315{
316 const struct cxsr_latency *latency;
317 int i;
318
319 if (fsb == 0 || mem == 0)
320 return NULL;
321
322 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
323 latency = &cxsr_latency_table[i];
324 if (is_desktop == latency->is_desktop &&
325 is_ddr3 == latency->is_ddr3 &&
326 fsb == latency->fsb_freq && mem == latency->mem_freq)
327 return latency;
328 }
329
330 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
331
332 return NULL;
333}
334
335static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
336{
337 u32 val;
338
339 vlv_punit_get(dev_priv);
340
341 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
342 if (enable)
343 val &= ~FORCE_DDR_HIGH_FREQ;
344 else
345 val |= FORCE_DDR_HIGH_FREQ;
346 val &= ~FORCE_DDR_LOW_FREQ;
347 val |= FORCE_DDR_FREQ_REQ_ACK;
348 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
349
350 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
351 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
352 drm_err(&dev_priv->drm,
353 "timed out waiting for Punit DDR DVFS request\n");
354
355 vlv_punit_put(dev_priv);
356}
357
358static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
359{
360 u32 val;
361
362 vlv_punit_get(dev_priv);
363
364 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
365 if (enable)
366 val |= DSP_MAXFIFO_PM5_ENABLE;
367 else
368 val &= ~DSP_MAXFIFO_PM5_ENABLE;
369 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
370
371 vlv_punit_put(dev_priv);
372}
373
374#define FW_WM(value, plane) \
375 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
376
377static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
378{
379 bool was_enabled;
380 u32 val;
381
382 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
383 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
384 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
385 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
386 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
387 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
388 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
389 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
390 } else if (IS_PINEVIEW(dev_priv)) {
391 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
392 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
393 if (enable)
394 val |= PINEVIEW_SELF_REFRESH_EN;
395 else
396 val &= ~PINEVIEW_SELF_REFRESH_EN;
397 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
398 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
399 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
400 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
401 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
402 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
403 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
404 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
405 } else if (IS_I915GM(dev_priv)) {
406 /*
407 * FIXME can't find a bit like this for 915G, and
408 * and yet it does have the related watermark in
409 * FW_BLC_SELF. What's going on?
410 */
411 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
412 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
413 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
414 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
415 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
416 } else {
417 return false;
418 }
419
420 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
421
422 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
423 enableddisabled(enable),
424 enableddisabled(was_enabled));
425
426 return was_enabled;
427}
428
429/**
430 * intel_set_memory_cxsr - Configure CxSR state
431 * @dev_priv: i915 device
432 * @enable: Allow vs. disallow CxSR
433 *
434 * Allow or disallow the system to enter a special CxSR
435 * (C-state self refresh) state. What typically happens in CxSR mode
436 * is that several display FIFOs may get combined into a single larger
437 * FIFO for a particular plane (so called max FIFO mode) to allow the
438 * system to defer memory fetches longer, and the memory will enter
439 * self refresh.
440 *
441 * Note that enabling CxSR does not guarantee that the system enter
442 * this special mode, nor does it guarantee that the system stays
443 * in that mode once entered. So this just allows/disallows the system
444 * to autonomously utilize the CxSR mode. Other factors such as core
445 * C-states will affect when/if the system actually enters/exits the
446 * CxSR mode.
447 *
448 * Note that on VLV/CHV this actually only controls the max FIFO mode,
449 * and the system is free to enter/exit memory self refresh at any time
450 * even when the use of CxSR has been disallowed.
451 *
452 * While the system is actually in the CxSR/max FIFO mode, some plane
453 * control registers will not get latched on vblank. Thus in order to
454 * guarantee the system will respond to changes in the plane registers
455 * we must always disallow CxSR prior to making changes to those registers.
456 * Unfortunately the system will re-evaluate the CxSR conditions at
457 * frame start which happens after vblank start (which is when the plane
458 * registers would get latched), so we can't proceed with the plane update
459 * during the same frame where we disallowed CxSR.
460 *
461 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
462 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
463 * the hardware w.r.t. HPLL SR when writing to plane registers.
464 * Disallowing just CxSR is sufficient.
465 */
466bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
467{
468 bool ret;
469
470 mutex_lock(&dev_priv->wm.wm_mutex);
471 ret = _intel_set_memory_cxsr(dev_priv, enable);
472 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
473 dev_priv->wm.vlv.cxsr = enable;
474 else if (IS_G4X(dev_priv))
475 dev_priv->wm.g4x.cxsr = enable;
476 mutex_unlock(&dev_priv->wm.wm_mutex);
477
478 return ret;
479}
480
481/*
482 * Latency for FIFO fetches is dependent on several factors:
483 * - memory configuration (speed, channels)
484 * - chipset
485 * - current MCH state
486 * It can be fairly high in some situations, so here we assume a fairly
487 * pessimal value. It's a tradeoff between extra memory fetches (if we
488 * set this value too high, the FIFO will fetch frequently to stay full)
489 * and power consumption (set it too low to save power and we might see
490 * FIFO underruns and display "flicker").
491 *
492 * A value of 5us seems to be a good balance; safe for very low end
493 * platforms but not overly aggressive on lower latency configs.
494 */
495static const int pessimal_latency_ns = 5000;
496
497#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
498 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
499
500static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
501{
502 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
503 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
504 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
505 enum pipe pipe = crtc->pipe;
506 int sprite0_start, sprite1_start;
507 u32 dsparb, dsparb2, dsparb3;
508
509 switch (pipe) {
510 case PIPE_A:
511 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
512 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
513 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
514 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
515 break;
516 case PIPE_B:
517 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
518 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
519 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
520 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
521 break;
522 case PIPE_C:
523 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
524 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
525 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
526 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
527 break;
528 default:
529 MISSING_CASE(pipe);
530 return;
531 }
532
533 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
534 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
535 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
536 fifo_state->plane[PLANE_CURSOR] = 63;
537}
538
539static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
540 enum i9xx_plane_id i9xx_plane)
541{
542 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
543 int size;
544
545 size = dsparb & 0x7f;
546 if (i9xx_plane == PLANE_B)
547 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
548
549 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
550 dsparb, plane_name(i9xx_plane), size);
551
552 return size;
553}
554
555static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
556 enum i9xx_plane_id i9xx_plane)
557{
558 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
559 int size;
560
561 size = dsparb & 0x1ff;
562 if (i9xx_plane == PLANE_B)
563 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
564 size >>= 1; /* Convert to cachelines */
565
566 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
567 dsparb, plane_name(i9xx_plane), size);
568
569 return size;
570}
571
572static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
573 enum i9xx_plane_id i9xx_plane)
574{
575 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
576 int size;
577
578 size = dsparb & 0x7f;
579 size >>= 2; /* Convert to cachelines */
580
581 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
582 dsparb, plane_name(i9xx_plane), size);
583
584 return size;
585}
586
587/* Pineview has different values for various configs */
588static const struct intel_watermark_params pnv_display_wm = {
589 .fifo_size = PINEVIEW_DISPLAY_FIFO,
590 .max_wm = PINEVIEW_MAX_WM,
591 .default_wm = PINEVIEW_DFT_WM,
592 .guard_size = PINEVIEW_GUARD_WM,
593 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594};
595
596static const struct intel_watermark_params pnv_display_hplloff_wm = {
597 .fifo_size = PINEVIEW_DISPLAY_FIFO,
598 .max_wm = PINEVIEW_MAX_WM,
599 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
600 .guard_size = PINEVIEW_GUARD_WM,
601 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
602};
603
604static const struct intel_watermark_params pnv_cursor_wm = {
605 .fifo_size = PINEVIEW_CURSOR_FIFO,
606 .max_wm = PINEVIEW_CURSOR_MAX_WM,
607 .default_wm = PINEVIEW_CURSOR_DFT_WM,
608 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
609 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
610};
611
612static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
613 .fifo_size = PINEVIEW_CURSOR_FIFO,
614 .max_wm = PINEVIEW_CURSOR_MAX_WM,
615 .default_wm = PINEVIEW_CURSOR_DFT_WM,
616 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
617 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
618};
619
620static const struct intel_watermark_params i965_cursor_wm_info = {
621 .fifo_size = I965_CURSOR_FIFO,
622 .max_wm = I965_CURSOR_MAX_WM,
623 .default_wm = I965_CURSOR_DFT_WM,
624 .guard_size = 2,
625 .cacheline_size = I915_FIFO_LINE_SIZE,
626};
627
628static const struct intel_watermark_params i945_wm_info = {
629 .fifo_size = I945_FIFO_SIZE,
630 .max_wm = I915_MAX_WM,
631 .default_wm = 1,
632 .guard_size = 2,
633 .cacheline_size = I915_FIFO_LINE_SIZE,
634};
635
636static const struct intel_watermark_params i915_wm_info = {
637 .fifo_size = I915_FIFO_SIZE,
638 .max_wm = I915_MAX_WM,
639 .default_wm = 1,
640 .guard_size = 2,
641 .cacheline_size = I915_FIFO_LINE_SIZE,
642};
643
644static const struct intel_watermark_params i830_a_wm_info = {
645 .fifo_size = I855GM_FIFO_SIZE,
646 .max_wm = I915_MAX_WM,
647 .default_wm = 1,
648 .guard_size = 2,
649 .cacheline_size = I830_FIFO_LINE_SIZE,
650};
651
652static const struct intel_watermark_params i830_bc_wm_info = {
653 .fifo_size = I855GM_FIFO_SIZE,
654 .max_wm = I915_MAX_WM/2,
655 .default_wm = 1,
656 .guard_size = 2,
657 .cacheline_size = I830_FIFO_LINE_SIZE,
658};
659
660static const struct intel_watermark_params i845_wm_info = {
661 .fifo_size = I830_FIFO_SIZE,
662 .max_wm = I915_MAX_WM,
663 .default_wm = 1,
664 .guard_size = 2,
665 .cacheline_size = I830_FIFO_LINE_SIZE,
666};
667
668/**
669 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @cpp: Plane bytes per pixel
672 * @latency: Memory wakeup latency in 0.1us units
673 *
674 * Compute the watermark using the method 1 or "small buffer"
675 * formula. The caller may additonally add extra cachelines
676 * to account for TLB misses and clock crossings.
677 *
678 * This method is concerned with the short term drain rate
679 * of the FIFO, ie. it does not account for blanking periods
680 * which would effectively reduce the average drain rate across
681 * a longer period. The name "small" refers to the fact the
682 * FIFO is relatively small compared to the amount of data
683 * fetched.
684 *
685 * The FIFO level vs. time graph might look something like:
686 *
687 * |\ |\
688 * | \ | \
689 * __---__---__ (- plane active, _ blanking)
690 * -> time
691 *
692 * or perhaps like this:
693 *
694 * |\|\ |\|\
695 * __----__----__ (- plane active, _ blanking)
696 * -> time
697 *
698 * Returns:
699 * The watermark in bytes
700 */
701static unsigned int intel_wm_method1(unsigned int pixel_rate,
702 unsigned int cpp,
703 unsigned int latency)
704{
705 u64 ret;
706
707 ret = mul_u32_u32(pixel_rate, cpp * latency);
708 ret = DIV_ROUND_UP_ULL(ret, 10000);
709
710 return ret;
711}
712
713/**
714 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
715 * @pixel_rate: Pipe pixel rate in kHz
716 * @htotal: Pipe horizontal total
717 * @width: Plane width in pixels
718 * @cpp: Plane bytes per pixel
719 * @latency: Memory wakeup latency in 0.1us units
720 *
721 * Compute the watermark using the method 2 or "large buffer"
722 * formula. The caller may additonally add extra cachelines
723 * to account for TLB misses and clock crossings.
724 *
725 * This method is concerned with the long term drain rate
726 * of the FIFO, ie. it does account for blanking periods
727 * which effectively reduce the average drain rate across
728 * a longer period. The name "large" refers to the fact the
729 * FIFO is relatively large compared to the amount of data
730 * fetched.
731 *
732 * The FIFO level vs. time graph might look something like:
733 *
734 * |\___ |\___
735 * | \___ | \___
736 * | \ | \
737 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
738 * -> time
739 *
740 * Returns:
741 * The watermark in bytes
742 */
743static unsigned int intel_wm_method2(unsigned int pixel_rate,
744 unsigned int htotal,
745 unsigned int width,
746 unsigned int cpp,
747 unsigned int latency)
748{
749 unsigned int ret;
750
751 /*
752 * FIXME remove once all users are computing
753 * watermarks in the correct place.
754 */
755 if (WARN_ON_ONCE(htotal == 0))
756 htotal = 1;
757
758 ret = (latency * pixel_rate) / (htotal * 10000);
759 ret = (ret + 1) * width * cpp;
760
761 return ret;
762}
763
764/**
765 * intel_calculate_wm - calculate watermark level
766 * @pixel_rate: pixel clock
767 * @wm: chip FIFO params
768 * @fifo_size: size of the FIFO buffer
769 * @cpp: bytes per pixel
770 * @latency_ns: memory latency for the platform
771 *
772 * Calculate the watermark level (the level at which the display plane will
773 * start fetching from memory again). Each chip has a different display
774 * FIFO size and allocation, so the caller needs to figure that out and pass
775 * in the correct intel_watermark_params structure.
776 *
777 * As the pixel clock runs, the FIFO will be drained at a rate that depends
778 * on the pixel size. When it reaches the watermark level, it'll start
779 * fetching FIFO line sized based chunks from memory until the FIFO fills
780 * past the watermark point. If the FIFO drains completely, a FIFO underrun
781 * will occur, and a display engine hang could result.
782 */
783static unsigned int intel_calculate_wm(int pixel_rate,
784 const struct intel_watermark_params *wm,
785 int fifo_size, int cpp,
786 unsigned int latency_ns)
787{
788 int entries, wm_size;
789
790 /*
791 * Note: we need to make sure we don't overflow for various clock &
792 * latency values.
793 * clocks go from a few thousand to several hundred thousand.
794 * latency is usually a few thousand
795 */
796 entries = intel_wm_method1(pixel_rate, cpp,
797 latency_ns / 100);
798 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
799 wm->guard_size;
800 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
801
802 wm_size = fifo_size - entries;
803 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
804
805 /* Don't promote wm_size to unsigned... */
806 if (wm_size > wm->max_wm)
807 wm_size = wm->max_wm;
808 if (wm_size <= 0)
809 wm_size = wm->default_wm;
810
811 /*
812 * Bspec seems to indicate that the value shouldn't be lower than
813 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
814 * Lets go for 8 which is the burst size since certain platforms
815 * already use a hardcoded 8 (which is what the spec says should be
816 * done).
817 */
818 if (wm_size <= 8)
819 wm_size = 8;
820
821 return wm_size;
822}
823
824static bool is_disabling(int old, int new, int threshold)
825{
826 return old >= threshold && new < threshold;
827}
828
829static bool is_enabling(int old, int new, int threshold)
830{
831 return old < threshold && new >= threshold;
832}
833
834static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
835{
836 return dev_priv->wm.max_level + 1;
837}
838
839static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
840 const struct intel_plane_state *plane_state)
841{
842 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
843
844 /* FIXME check the 'enable' instead */
845 if (!crtc_state->hw.active)
846 return false;
847
848 /*
849 * Treat cursor with fb as always visible since cursor updates
850 * can happen faster than the vrefresh rate, and the current
851 * watermark code doesn't handle that correctly. Cursor updates
852 * which set/clear the fb or change the cursor size are going
853 * to get throttled by intel_legacy_cursor_update() to work
854 * around this problem with the watermark code.
855 */
856 if (plane->id == PLANE_CURSOR)
857 return plane_state->hw.fb != NULL;
858 else
859 return plane_state->uapi.visible;
860}
861
862static bool intel_crtc_active(struct intel_crtc *crtc)
863{
864 /* Be paranoid as we can arrive here with only partial
865 * state retrieved from the hardware during setup.
866 *
867 * We can ditch the adjusted_mode.crtc_clock check as soon
868 * as Haswell has gained clock readout/fastboot support.
869 *
870 * We can ditch the crtc->primary->state->fb check as soon as we can
871 * properly reconstruct framebuffers.
872 *
873 * FIXME: The intel_crtc->active here should be switched to
874 * crtc->state->active once we have proper CRTC states wired up
875 * for atomic.
876 */
877 return crtc->active && crtc->base.primary->state->fb &&
878 crtc->config->hw.adjusted_mode.crtc_clock;
879}
880
881static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
882{
883 struct intel_crtc *crtc, *enabled = NULL;
884
885 for_each_intel_crtc(&dev_priv->drm, crtc) {
886 if (intel_crtc_active(crtc)) {
887 if (enabled)
888 return NULL;
889 enabled = crtc;
890 }
891 }
892
893 return enabled;
894}
895
896static void pnv_update_wm(struct intel_crtc *unused_crtc)
897{
898 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
899 struct intel_crtc *crtc;
900 const struct cxsr_latency *latency;
901 u32 reg;
902 unsigned int wm;
903
904 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
905 dev_priv->is_ddr3,
906 dev_priv->fsb_freq,
907 dev_priv->mem_freq);
908 if (!latency) {
909 drm_dbg_kms(&dev_priv->drm,
910 "Unknown FSB/MEM found, disable CxSR\n");
911 intel_set_memory_cxsr(dev_priv, false);
912 return;
913 }
914
915 crtc = single_enabled_crtc(dev_priv);
916 if (crtc) {
917 const struct drm_display_mode *pipe_mode =
918 &crtc->config->hw.pipe_mode;
919 const struct drm_framebuffer *fb =
920 crtc->base.primary->state->fb;
921 int cpp = fb->format->cpp[0];
922 int clock = pipe_mode->crtc_clock;
923
924 /* Display SR */
925 wm = intel_calculate_wm(clock, &pnv_display_wm,
926 pnv_display_wm.fifo_size,
927 cpp, latency->display_sr);
928 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
929 reg &= ~DSPFW_SR_MASK;
930 reg |= FW_WM(wm, SR);
931 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
932 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
933
934 /* cursor SR */
935 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
936 pnv_display_wm.fifo_size,
937 4, latency->cursor_sr);
938 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
939 reg &= ~DSPFW_CURSOR_SR_MASK;
940 reg |= FW_WM(wm, CURSOR_SR);
941 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
942
943 /* Display HPLL off SR */
944 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
945 pnv_display_hplloff_wm.fifo_size,
946 cpp, latency->display_hpll_disable);
947 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
948 reg &= ~DSPFW_HPLL_SR_MASK;
949 reg |= FW_WM(wm, HPLL_SR);
950 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
951
952 /* cursor HPLL off SR */
953 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
954 pnv_display_hplloff_wm.fifo_size,
955 4, latency->cursor_hpll_disable);
956 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
957 reg &= ~DSPFW_HPLL_CURSOR_MASK;
958 reg |= FW_WM(wm, HPLL_CURSOR);
959 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
960 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
961
962 intel_set_memory_cxsr(dev_priv, true);
963 } else {
964 intel_set_memory_cxsr(dev_priv, false);
965 }
966}
967
968/*
969 * Documentation says:
970 * "If the line size is small, the TLB fetches can get in the way of the
971 * data fetches, causing some lag in the pixel data return which is not
972 * accounted for in the above formulas. The following adjustment only
973 * needs to be applied if eight whole lines fit in the buffer at once.
974 * The WM is adjusted upwards by the difference between the FIFO size
975 * and the size of 8 whole lines. This adjustment is always performed
976 * in the actual pixel depth regardless of whether FBC is enabled or not."
977 */
978static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
979{
980 int tlb_miss = fifo_size * 64 - width * cpp * 8;
981
982 return max(0, tlb_miss);
983}
984
985static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
986 const struct g4x_wm_values *wm)
987{
988 enum pipe pipe;
989
990 for_each_pipe(dev_priv, pipe)
991 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
992
993 intel_uncore_write(&dev_priv->uncore, DSPFW1,
994 FW_WM(wm->sr.plane, SR) |
995 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
996 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
997 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
998 intel_uncore_write(&dev_priv->uncore, DSPFW2,
999 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
1000 FW_WM(wm->sr.fbc, FBC_SR) |
1001 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1002 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1003 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1004 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1005 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1006 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1007 FW_WM(wm->sr.cursor, CURSOR_SR) |
1008 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1009 FW_WM(wm->hpll.plane, HPLL_SR));
1010
1011 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1012}
1013
1014#define FW_WM_VLV(value, plane) \
1015 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1016
1017static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1018 const struct vlv_wm_values *wm)
1019{
1020 enum pipe pipe;
1021
1022 for_each_pipe(dev_priv, pipe) {
1023 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1024
1025 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1026 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1027 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1028 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1029 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1030 }
1031
1032 /*
1033 * Zero the (unused) WM1 watermarks, and also clear all the
1034 * high order bits so that there are no out of bounds values
1035 * present in the registers during the reprogramming.
1036 */
1037 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1039 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1040 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1041 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1042
1043 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1044 FW_WM(wm->sr.plane, SR) |
1045 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1046 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1047 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1048 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1049 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1050 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1051 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1052 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1053 FW_WM(wm->sr.cursor, CURSOR_SR));
1054
1055 if (IS_CHERRYVIEW(dev_priv)) {
1056 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1057 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1058 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1059 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1060 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1061 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1062 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1063 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1065 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1066 FW_WM(wm->sr.plane >> 9, SR_HI) |
1067 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1068 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1069 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1072 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1075 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1076 } else {
1077 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1078 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1079 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1080 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1081 FW_WM(wm->sr.plane >> 9, SR_HI) |
1082 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1083 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1084 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1085 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1086 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1087 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1088 }
1089
1090 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1091}
1092
1093#undef FW_WM_VLV
1094
1095static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1096{
1097 /* all latencies in usec */
1098 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1099 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1100 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1101
1102 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1103}
1104
1105static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1106{
1107 /*
1108 * DSPCNTR[13] supposedly controls whether the
1109 * primary plane can use the FIFO space otherwise
1110 * reserved for the sprite plane. It's not 100% clear
1111 * what the actual FIFO size is, but it looks like we
1112 * can happily set both primary and sprite watermarks
1113 * up to 127 cachelines. So that would seem to mean
1114 * that either DSPCNTR[13] doesn't do anything, or that
1115 * the total FIFO is >= 256 cachelines in size. Either
1116 * way, we don't seem to have to worry about this
1117 * repartitioning as the maximum watermark value the
1118 * register can hold for each plane is lower than the
1119 * minimum FIFO size.
1120 */
1121 switch (plane_id) {
1122 case PLANE_CURSOR:
1123 return 63;
1124 case PLANE_PRIMARY:
1125 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1126 case PLANE_SPRITE0:
1127 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1128 default:
1129 MISSING_CASE(plane_id);
1130 return 0;
1131 }
1132}
1133
1134static int g4x_fbc_fifo_size(int level)
1135{
1136 switch (level) {
1137 case G4X_WM_LEVEL_SR:
1138 return 7;
1139 case G4X_WM_LEVEL_HPLL:
1140 return 15;
1141 default:
1142 MISSING_CASE(level);
1143 return 0;
1144 }
1145}
1146
1147static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1148 const struct intel_plane_state *plane_state,
1149 int level)
1150{
1151 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1152 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1153 const struct drm_display_mode *pipe_mode =
1154 &crtc_state->hw.pipe_mode;
1155 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1156 unsigned int clock, htotal, cpp, width, wm;
1157
1158 if (latency == 0)
1159 return USHRT_MAX;
1160
1161 if (!intel_wm_plane_visible(crtc_state, plane_state))
1162 return 0;
1163
1164 cpp = plane_state->hw.fb->format->cpp[0];
1165
1166 /*
1167 * Not 100% sure which way ELK should go here as the
1168 * spec only says CL/CTG should assume 32bpp and BW
1169 * doesn't need to. But as these things followed the
1170 * mobile vs. desktop lines on gen3 as well, let's
1171 * assume ELK doesn't need this.
1172 *
1173 * The spec also fails to list such a restriction for
1174 * the HPLL watermark, which seems a little strange.
1175 * Let's use 32bpp for the HPLL watermark as well.
1176 */
1177 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1178 level != G4X_WM_LEVEL_NORMAL)
1179 cpp = max(cpp, 4u);
1180
1181 clock = pipe_mode->crtc_clock;
1182 htotal = pipe_mode->crtc_htotal;
1183
1184 width = drm_rect_width(&plane_state->uapi.dst);
1185
1186 if (plane->id == PLANE_CURSOR) {
1187 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1188 } else if (plane->id == PLANE_PRIMARY &&
1189 level == G4X_WM_LEVEL_NORMAL) {
1190 wm = intel_wm_method1(clock, cpp, latency);
1191 } else {
1192 unsigned int small, large;
1193
1194 small = intel_wm_method1(clock, cpp, latency);
1195 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1196
1197 wm = min(small, large);
1198 }
1199
1200 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1201 width, cpp);
1202
1203 wm = DIV_ROUND_UP(wm, 64) + 2;
1204
1205 return min_t(unsigned int, wm, USHRT_MAX);
1206}
1207
1208static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1209 int level, enum plane_id plane_id, u16 value)
1210{
1211 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1212 bool dirty = false;
1213
1214 for (; level < intel_wm_num_levels(dev_priv); level++) {
1215 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1216
1217 dirty |= raw->plane[plane_id] != value;
1218 raw->plane[plane_id] = value;
1219 }
1220
1221 return dirty;
1222}
1223
1224static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1225 int level, u16 value)
1226{
1227 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1228 bool dirty = false;
1229
1230 /* NORMAL level doesn't have an FBC watermark */
1231 level = max(level, G4X_WM_LEVEL_SR);
1232
1233 for (; level < intel_wm_num_levels(dev_priv); level++) {
1234 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1235
1236 dirty |= raw->fbc != value;
1237 raw->fbc = value;
1238 }
1239
1240 return dirty;
1241}
1242
1243static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1244 const struct intel_plane_state *plane_state,
1245 u32 pri_val);
1246
1247static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1248 const struct intel_plane_state *plane_state)
1249{
1250 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1251 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1252 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1253 enum plane_id plane_id = plane->id;
1254 bool dirty = false;
1255 int level;
1256
1257 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1258 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1259 if (plane_id == PLANE_PRIMARY)
1260 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1261 goto out;
1262 }
1263
1264 for (level = 0; level < num_levels; level++) {
1265 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1266 int wm, max_wm;
1267
1268 wm = g4x_compute_wm(crtc_state, plane_state, level);
1269 max_wm = g4x_plane_fifo_size(plane_id, level);
1270
1271 if (wm > max_wm)
1272 break;
1273
1274 dirty |= raw->plane[plane_id] != wm;
1275 raw->plane[plane_id] = wm;
1276
1277 if (plane_id != PLANE_PRIMARY ||
1278 level == G4X_WM_LEVEL_NORMAL)
1279 continue;
1280
1281 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1282 raw->plane[plane_id]);
1283 max_wm = g4x_fbc_fifo_size(level);
1284
1285 /*
1286 * FBC wm is not mandatory as we
1287 * can always just disable its use.
1288 */
1289 if (wm > max_wm)
1290 wm = USHRT_MAX;
1291
1292 dirty |= raw->fbc != wm;
1293 raw->fbc = wm;
1294 }
1295
1296 /* mark watermarks as invalid */
1297 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1298
1299 if (plane_id == PLANE_PRIMARY)
1300 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1301
1302 out:
1303 if (dirty) {
1304 drm_dbg_kms(&dev_priv->drm,
1305 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1306 plane->base.name,
1307 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1308 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1309 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1310
1311 if (plane_id == PLANE_PRIMARY)
1312 drm_dbg_kms(&dev_priv->drm,
1313 "FBC watermarks: SR=%d, HPLL=%d\n",
1314 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1315 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1316 }
1317
1318 return dirty;
1319}
1320
1321static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1322 enum plane_id plane_id, int level)
1323{
1324 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1325
1326 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1327}
1328
1329static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1330 int level)
1331{
1332 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1333
1334 if (level > dev_priv->wm.max_level)
1335 return false;
1336
1337 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1338 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1339 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1340}
1341
1342/* mark all levels starting from 'level' as invalid */
1343static void g4x_invalidate_wms(struct intel_crtc *crtc,
1344 struct g4x_wm_state *wm_state, int level)
1345{
1346 if (level <= G4X_WM_LEVEL_NORMAL) {
1347 enum plane_id plane_id;
1348
1349 for_each_plane_id_on_crtc(crtc, plane_id)
1350 wm_state->wm.plane[plane_id] = USHRT_MAX;
1351 }
1352
1353 if (level <= G4X_WM_LEVEL_SR) {
1354 wm_state->cxsr = false;
1355 wm_state->sr.cursor = USHRT_MAX;
1356 wm_state->sr.plane = USHRT_MAX;
1357 wm_state->sr.fbc = USHRT_MAX;
1358 }
1359
1360 if (level <= G4X_WM_LEVEL_HPLL) {
1361 wm_state->hpll_en = false;
1362 wm_state->hpll.cursor = USHRT_MAX;
1363 wm_state->hpll.plane = USHRT_MAX;
1364 wm_state->hpll.fbc = USHRT_MAX;
1365 }
1366}
1367
1368static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1369 int level)
1370{
1371 if (level < G4X_WM_LEVEL_SR)
1372 return false;
1373
1374 if (level >= G4X_WM_LEVEL_SR &&
1375 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1376 return false;
1377
1378 if (level >= G4X_WM_LEVEL_HPLL &&
1379 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1380 return false;
1381
1382 return true;
1383}
1384
1385static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1386{
1387 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1388 struct intel_atomic_state *state =
1389 to_intel_atomic_state(crtc_state->uapi.state);
1390 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1391 int num_active_planes = hweight8(crtc_state->active_planes &
1392 ~BIT(PLANE_CURSOR));
1393 const struct g4x_pipe_wm *raw;
1394 const struct intel_plane_state *old_plane_state;
1395 const struct intel_plane_state *new_plane_state;
1396 struct intel_plane *plane;
1397 enum plane_id plane_id;
1398 int i, level;
1399 unsigned int dirty = 0;
1400
1401 for_each_oldnew_intel_plane_in_state(state, plane,
1402 old_plane_state,
1403 new_plane_state, i) {
1404 if (new_plane_state->hw.crtc != &crtc->base &&
1405 old_plane_state->hw.crtc != &crtc->base)
1406 continue;
1407
1408 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1409 dirty |= BIT(plane->id);
1410 }
1411
1412 if (!dirty)
1413 return 0;
1414
1415 level = G4X_WM_LEVEL_NORMAL;
1416 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1417 goto out;
1418
1419 raw = &crtc_state->wm.g4x.raw[level];
1420 for_each_plane_id_on_crtc(crtc, plane_id)
1421 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1422
1423 level = G4X_WM_LEVEL_SR;
1424 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1425 goto out;
1426
1427 raw = &crtc_state->wm.g4x.raw[level];
1428 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1429 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1430 wm_state->sr.fbc = raw->fbc;
1431
1432 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1433
1434 level = G4X_WM_LEVEL_HPLL;
1435 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1436 goto out;
1437
1438 raw = &crtc_state->wm.g4x.raw[level];
1439 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1440 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1441 wm_state->hpll.fbc = raw->fbc;
1442
1443 wm_state->hpll_en = wm_state->cxsr;
1444
1445 level++;
1446
1447 out:
1448 if (level == G4X_WM_LEVEL_NORMAL)
1449 return -EINVAL;
1450
1451 /* invalidate the higher levels */
1452 g4x_invalidate_wms(crtc, wm_state, level);
1453
1454 /*
1455 * Determine if the FBC watermark(s) can be used. IF
1456 * this isn't the case we prefer to disable the FBC
1457 * watermark(s) rather than disable the SR/HPLL
1458 * level(s) entirely. 'level-1' is the highest valid
1459 * level here.
1460 */
1461 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1462
1463 return 0;
1464}
1465
1466static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1467{
1468 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1469 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1470 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1471 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1472 struct intel_atomic_state *intel_state =
1473 to_intel_atomic_state(new_crtc_state->uapi.state);
1474 const struct intel_crtc_state *old_crtc_state =
1475 intel_atomic_get_old_crtc_state(intel_state, crtc);
1476 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1477 enum plane_id plane_id;
1478
1479 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1480 *intermediate = *optimal;
1481
1482 intermediate->cxsr = false;
1483 intermediate->hpll_en = false;
1484 goto out;
1485 }
1486
1487 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1488 !new_crtc_state->disable_cxsr;
1489 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1490 !new_crtc_state->disable_cxsr;
1491 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1492
1493 for_each_plane_id_on_crtc(crtc, plane_id) {
1494 intermediate->wm.plane[plane_id] =
1495 max(optimal->wm.plane[plane_id],
1496 active->wm.plane[plane_id]);
1497
1498 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1499 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1500 }
1501
1502 intermediate->sr.plane = max(optimal->sr.plane,
1503 active->sr.plane);
1504 intermediate->sr.cursor = max(optimal->sr.cursor,
1505 active->sr.cursor);
1506 intermediate->sr.fbc = max(optimal->sr.fbc,
1507 active->sr.fbc);
1508
1509 intermediate->hpll.plane = max(optimal->hpll.plane,
1510 active->hpll.plane);
1511 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1512 active->hpll.cursor);
1513 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1514 active->hpll.fbc);
1515
1516 drm_WARN_ON(&dev_priv->drm,
1517 (intermediate->sr.plane >
1518 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1519 intermediate->sr.cursor >
1520 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1521 intermediate->cxsr);
1522 drm_WARN_ON(&dev_priv->drm,
1523 (intermediate->sr.plane >
1524 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1525 intermediate->sr.cursor >
1526 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1527 intermediate->hpll_en);
1528
1529 drm_WARN_ON(&dev_priv->drm,
1530 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1531 intermediate->fbc_en && intermediate->cxsr);
1532 drm_WARN_ON(&dev_priv->drm,
1533 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1534 intermediate->fbc_en && intermediate->hpll_en);
1535
1536out:
1537 /*
1538 * If our intermediate WM are identical to the final WM, then we can
1539 * omit the post-vblank programming; only update if it's different.
1540 */
1541 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1542 new_crtc_state->wm.need_postvbl_update = true;
1543
1544 return 0;
1545}
1546
1547static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1548 struct g4x_wm_values *wm)
1549{
1550 struct intel_crtc *crtc;
1551 int num_active_pipes = 0;
1552
1553 wm->cxsr = true;
1554 wm->hpll_en = true;
1555 wm->fbc_en = true;
1556
1557 for_each_intel_crtc(&dev_priv->drm, crtc) {
1558 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1559
1560 if (!crtc->active)
1561 continue;
1562
1563 if (!wm_state->cxsr)
1564 wm->cxsr = false;
1565 if (!wm_state->hpll_en)
1566 wm->hpll_en = false;
1567 if (!wm_state->fbc_en)
1568 wm->fbc_en = false;
1569
1570 num_active_pipes++;
1571 }
1572
1573 if (num_active_pipes != 1) {
1574 wm->cxsr = false;
1575 wm->hpll_en = false;
1576 wm->fbc_en = false;
1577 }
1578
1579 for_each_intel_crtc(&dev_priv->drm, crtc) {
1580 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1581 enum pipe pipe = crtc->pipe;
1582
1583 wm->pipe[pipe] = wm_state->wm;
1584 if (crtc->active && wm->cxsr)
1585 wm->sr = wm_state->sr;
1586 if (crtc->active && wm->hpll_en)
1587 wm->hpll = wm_state->hpll;
1588 }
1589}
1590
1591static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1592{
1593 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1594 struct g4x_wm_values new_wm = {};
1595
1596 g4x_merge_wm(dev_priv, &new_wm);
1597
1598 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1599 return;
1600
1601 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1602 _intel_set_memory_cxsr(dev_priv, false);
1603
1604 g4x_write_wm_values(dev_priv, &new_wm);
1605
1606 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1607 _intel_set_memory_cxsr(dev_priv, true);
1608
1609 *old_wm = new_wm;
1610}
1611
1612static void g4x_initial_watermarks(struct intel_atomic_state *state,
1613 struct intel_crtc *crtc)
1614{
1615 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1616 const struct intel_crtc_state *crtc_state =
1617 intel_atomic_get_new_crtc_state(state, crtc);
1618
1619 mutex_lock(&dev_priv->wm.wm_mutex);
1620 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1621 g4x_program_watermarks(dev_priv);
1622 mutex_unlock(&dev_priv->wm.wm_mutex);
1623}
1624
1625static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1626 struct intel_crtc *crtc)
1627{
1628 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1629 const struct intel_crtc_state *crtc_state =
1630 intel_atomic_get_new_crtc_state(state, crtc);
1631
1632 if (!crtc_state->wm.need_postvbl_update)
1633 return;
1634
1635 mutex_lock(&dev_priv->wm.wm_mutex);
1636 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1637 g4x_program_watermarks(dev_priv);
1638 mutex_unlock(&dev_priv->wm.wm_mutex);
1639}
1640
1641/* latency must be in 0.1us units. */
1642static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1643 unsigned int htotal,
1644 unsigned int width,
1645 unsigned int cpp,
1646 unsigned int latency)
1647{
1648 unsigned int ret;
1649
1650 ret = intel_wm_method2(pixel_rate, htotal,
1651 width, cpp, latency);
1652 ret = DIV_ROUND_UP(ret, 64);
1653
1654 return ret;
1655}
1656
1657static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1658{
1659 /* all latencies in usec */
1660 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1661
1662 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1663
1664 if (IS_CHERRYVIEW(dev_priv)) {
1665 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1666 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1667
1668 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1669 }
1670}
1671
1672static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1673 const struct intel_plane_state *plane_state,
1674 int level)
1675{
1676 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1677 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1678 const struct drm_display_mode *pipe_mode =
1679 &crtc_state->hw.pipe_mode;
1680 unsigned int clock, htotal, cpp, width, wm;
1681
1682 if (dev_priv->wm.pri_latency[level] == 0)
1683 return USHRT_MAX;
1684
1685 if (!intel_wm_plane_visible(crtc_state, plane_state))
1686 return 0;
1687
1688 cpp = plane_state->hw.fb->format->cpp[0];
1689 clock = pipe_mode->crtc_clock;
1690 htotal = pipe_mode->crtc_htotal;
1691 width = crtc_state->pipe_src_w;
1692
1693 if (plane->id == PLANE_CURSOR) {
1694 /*
1695 * FIXME the formula gives values that are
1696 * too big for the cursor FIFO, and hence we
1697 * would never be able to use cursors. For
1698 * now just hardcode the watermark.
1699 */
1700 wm = 63;
1701 } else {
1702 wm = vlv_wm_method2(clock, htotal, width, cpp,
1703 dev_priv->wm.pri_latency[level] * 10);
1704 }
1705
1706 return min_t(unsigned int, wm, USHRT_MAX);
1707}
1708
1709static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1710{
1711 return (active_planes & (BIT(PLANE_SPRITE0) |
1712 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1713}
1714
1715static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1716{
1717 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1718 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1719 const struct g4x_pipe_wm *raw =
1720 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1721 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1722 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1723 int num_active_planes = hweight8(active_planes);
1724 const int fifo_size = 511;
1725 int fifo_extra, fifo_left = fifo_size;
1726 int sprite0_fifo_extra = 0;
1727 unsigned int total_rate;
1728 enum plane_id plane_id;
1729
1730 /*
1731 * When enabling sprite0 after sprite1 has already been enabled
1732 * we tend to get an underrun unless sprite0 already has some
1733 * FIFO space allcoated. Hence we always allocate at least one
1734 * cacheline for sprite0 whenever sprite1 is enabled.
1735 *
1736 * All other plane enable sequences appear immune to this problem.
1737 */
1738 if (vlv_need_sprite0_fifo_workaround(active_planes))
1739 sprite0_fifo_extra = 1;
1740
1741 total_rate = raw->plane[PLANE_PRIMARY] +
1742 raw->plane[PLANE_SPRITE0] +
1743 raw->plane[PLANE_SPRITE1] +
1744 sprite0_fifo_extra;
1745
1746 if (total_rate > fifo_size)
1747 return -EINVAL;
1748
1749 if (total_rate == 0)
1750 total_rate = 1;
1751
1752 for_each_plane_id_on_crtc(crtc, plane_id) {
1753 unsigned int rate;
1754
1755 if ((active_planes & BIT(plane_id)) == 0) {
1756 fifo_state->plane[plane_id] = 0;
1757 continue;
1758 }
1759
1760 rate = raw->plane[plane_id];
1761 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1762 fifo_left -= fifo_state->plane[plane_id];
1763 }
1764
1765 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1766 fifo_left -= sprite0_fifo_extra;
1767
1768 fifo_state->plane[PLANE_CURSOR] = 63;
1769
1770 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1771
1772 /* spread the remainder evenly */
1773 for_each_plane_id_on_crtc(crtc, plane_id) {
1774 int plane_extra;
1775
1776 if (fifo_left == 0)
1777 break;
1778
1779 if ((active_planes & BIT(plane_id)) == 0)
1780 continue;
1781
1782 plane_extra = min(fifo_extra, fifo_left);
1783 fifo_state->plane[plane_id] += plane_extra;
1784 fifo_left -= plane_extra;
1785 }
1786
1787 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1788
1789 /* give it all to the first plane if none are active */
1790 if (active_planes == 0) {
1791 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1792 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1793 }
1794
1795 return 0;
1796}
1797
1798/* mark all levels starting from 'level' as invalid */
1799static void vlv_invalidate_wms(struct intel_crtc *crtc,
1800 struct vlv_wm_state *wm_state, int level)
1801{
1802 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1803
1804 for (; level < intel_wm_num_levels(dev_priv); level++) {
1805 enum plane_id plane_id;
1806
1807 for_each_plane_id_on_crtc(crtc, plane_id)
1808 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1809
1810 wm_state->sr[level].cursor = USHRT_MAX;
1811 wm_state->sr[level].plane = USHRT_MAX;
1812 }
1813}
1814
1815static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1816{
1817 if (wm > fifo_size)
1818 return USHRT_MAX;
1819 else
1820 return fifo_size - wm;
1821}
1822
1823/*
1824 * Starting from 'level' set all higher
1825 * levels to 'value' in the "raw" watermarks.
1826 */
1827static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1828 int level, enum plane_id plane_id, u16 value)
1829{
1830 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1831 int num_levels = intel_wm_num_levels(dev_priv);
1832 bool dirty = false;
1833
1834 for (; level < num_levels; level++) {
1835 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1836
1837 dirty |= raw->plane[plane_id] != value;
1838 raw->plane[plane_id] = value;
1839 }
1840
1841 return dirty;
1842}
1843
1844static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1845 const struct intel_plane_state *plane_state)
1846{
1847 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1848 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1849 enum plane_id plane_id = plane->id;
1850 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1851 int level;
1852 bool dirty = false;
1853
1854 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1855 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1856 goto out;
1857 }
1858
1859 for (level = 0; level < num_levels; level++) {
1860 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1861 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1862 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1863
1864 if (wm > max_wm)
1865 break;
1866
1867 dirty |= raw->plane[plane_id] != wm;
1868 raw->plane[plane_id] = wm;
1869 }
1870
1871 /* mark all higher levels as invalid */
1872 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1873
1874out:
1875 if (dirty)
1876 drm_dbg_kms(&dev_priv->drm,
1877 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1878 plane->base.name,
1879 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1880 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1881 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1882
1883 return dirty;
1884}
1885
1886static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1887 enum plane_id plane_id, int level)
1888{
1889 const struct g4x_pipe_wm *raw =
1890 &crtc_state->wm.vlv.raw[level];
1891 const struct vlv_fifo_state *fifo_state =
1892 &crtc_state->wm.vlv.fifo_state;
1893
1894 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1895}
1896
1897static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1898{
1899 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1900 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1901 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1902 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1903}
1904
1905static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1906{
1907 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1908 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1909 struct intel_atomic_state *state =
1910 to_intel_atomic_state(crtc_state->uapi.state);
1911 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1912 const struct vlv_fifo_state *fifo_state =
1913 &crtc_state->wm.vlv.fifo_state;
1914 int num_active_planes = hweight8(crtc_state->active_planes &
1915 ~BIT(PLANE_CURSOR));
1916 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1917 const struct intel_plane_state *old_plane_state;
1918 const struct intel_plane_state *new_plane_state;
1919 struct intel_plane *plane;
1920 enum plane_id plane_id;
1921 int level, ret, i;
1922 unsigned int dirty = 0;
1923
1924 for_each_oldnew_intel_plane_in_state(state, plane,
1925 old_plane_state,
1926 new_plane_state, i) {
1927 if (new_plane_state->hw.crtc != &crtc->base &&
1928 old_plane_state->hw.crtc != &crtc->base)
1929 continue;
1930
1931 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1932 dirty |= BIT(plane->id);
1933 }
1934
1935 /*
1936 * DSPARB registers may have been reset due to the
1937 * power well being turned off. Make sure we restore
1938 * them to a consistent state even if no primary/sprite
1939 * planes are initially active.
1940 */
1941 if (needs_modeset)
1942 crtc_state->fifo_changed = true;
1943
1944 if (!dirty)
1945 return 0;
1946
1947 /* cursor changes don't warrant a FIFO recompute */
1948 if (dirty & ~BIT(PLANE_CURSOR)) {
1949 const struct intel_crtc_state *old_crtc_state =
1950 intel_atomic_get_old_crtc_state(state, crtc);
1951 const struct vlv_fifo_state *old_fifo_state =
1952 &old_crtc_state->wm.vlv.fifo_state;
1953
1954 ret = vlv_compute_fifo(crtc_state);
1955 if (ret)
1956 return ret;
1957
1958 if (needs_modeset ||
1959 memcmp(old_fifo_state, fifo_state,
1960 sizeof(*fifo_state)) != 0)
1961 crtc_state->fifo_changed = true;
1962 }
1963
1964 /* initially allow all levels */
1965 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1966 /*
1967 * Note that enabling cxsr with no primary/sprite planes
1968 * enabled can wedge the pipe. Hence we only allow cxsr
1969 * with exactly one enabled primary/sprite plane.
1970 */
1971 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1972
1973 for (level = 0; level < wm_state->num_levels; level++) {
1974 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1975 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1976
1977 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1978 break;
1979
1980 for_each_plane_id_on_crtc(crtc, plane_id) {
1981 wm_state->wm[level].plane[plane_id] =
1982 vlv_invert_wm_value(raw->plane[plane_id],
1983 fifo_state->plane[plane_id]);
1984 }
1985
1986 wm_state->sr[level].plane =
1987 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1988 raw->plane[PLANE_SPRITE0],
1989 raw->plane[PLANE_SPRITE1]),
1990 sr_fifo_size);
1991
1992 wm_state->sr[level].cursor =
1993 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1994 63);
1995 }
1996
1997 if (level == 0)
1998 return -EINVAL;
1999
2000 /* limit to only levels we can actually handle */
2001 wm_state->num_levels = level;
2002
2003 /* invalidate the higher levels */
2004 vlv_invalidate_wms(crtc, wm_state, level);
2005
2006 return 0;
2007}
2008
2009#define VLV_FIFO(plane, value) \
2010 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2011
2012static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2013 struct intel_crtc *crtc)
2014{
2015 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2016 struct intel_uncore *uncore = &dev_priv->uncore;
2017 const struct intel_crtc_state *crtc_state =
2018 intel_atomic_get_new_crtc_state(state, crtc);
2019 const struct vlv_fifo_state *fifo_state =
2020 &crtc_state->wm.vlv.fifo_state;
2021 int sprite0_start, sprite1_start, fifo_size;
2022 u32 dsparb, dsparb2, dsparb3;
2023
2024 if (!crtc_state->fifo_changed)
2025 return;
2026
2027 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2028 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2029 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2030
2031 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2032 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2033
2034 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2035
2036 /*
2037 * uncore.lock serves a double purpose here. It allows us to
2038 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2039 * it protects the DSPARB registers from getting clobbered by
2040 * parallel updates from multiple pipes.
2041 *
2042 * intel_pipe_update_start() has already disabled interrupts
2043 * for us, so a plain spin_lock() is sufficient here.
2044 */
2045 spin_lock(&uncore->lock);
2046
2047 switch (crtc->pipe) {
2048 case PIPE_A:
2049 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2050 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2051
2052 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2053 VLV_FIFO(SPRITEB, 0xff));
2054 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2055 VLV_FIFO(SPRITEB, sprite1_start));
2056
2057 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2058 VLV_FIFO(SPRITEB_HI, 0x1));
2059 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2060 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2061
2062 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2063 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2064 break;
2065 case PIPE_B:
2066 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2067 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2068
2069 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2070 VLV_FIFO(SPRITED, 0xff));
2071 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2072 VLV_FIFO(SPRITED, sprite1_start));
2073
2074 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2075 VLV_FIFO(SPRITED_HI, 0xff));
2076 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2077 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2078
2079 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2080 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2081 break;
2082 case PIPE_C:
2083 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2084 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2085
2086 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2087 VLV_FIFO(SPRITEF, 0xff));
2088 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2089 VLV_FIFO(SPRITEF, sprite1_start));
2090
2091 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2092 VLV_FIFO(SPRITEF_HI, 0xff));
2093 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2094 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2095
2096 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2097 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2098 break;
2099 default:
2100 break;
2101 }
2102
2103 intel_uncore_posting_read_fw(uncore, DSPARB);
2104
2105 spin_unlock(&uncore->lock);
2106}
2107
2108#undef VLV_FIFO
2109
2110static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2111{
2112 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2113 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2114 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2115 struct intel_atomic_state *intel_state =
2116 to_intel_atomic_state(new_crtc_state->uapi.state);
2117 const struct intel_crtc_state *old_crtc_state =
2118 intel_atomic_get_old_crtc_state(intel_state, crtc);
2119 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2120 int level;
2121
2122 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2123 *intermediate = *optimal;
2124
2125 intermediate->cxsr = false;
2126 goto out;
2127 }
2128
2129 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2130 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2131 !new_crtc_state->disable_cxsr;
2132
2133 for (level = 0; level < intermediate->num_levels; level++) {
2134 enum plane_id plane_id;
2135
2136 for_each_plane_id_on_crtc(crtc, plane_id) {
2137 intermediate->wm[level].plane[plane_id] =
2138 min(optimal->wm[level].plane[plane_id],
2139 active->wm[level].plane[plane_id]);
2140 }
2141
2142 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2143 active->sr[level].plane);
2144 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2145 active->sr[level].cursor);
2146 }
2147
2148 vlv_invalidate_wms(crtc, intermediate, level);
2149
2150out:
2151 /*
2152 * If our intermediate WM are identical to the final WM, then we can
2153 * omit the post-vblank programming; only update if it's different.
2154 */
2155 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2156 new_crtc_state->wm.need_postvbl_update = true;
2157
2158 return 0;
2159}
2160
2161static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2162 struct vlv_wm_values *wm)
2163{
2164 struct intel_crtc *crtc;
2165 int num_active_pipes = 0;
2166
2167 wm->level = dev_priv->wm.max_level;
2168 wm->cxsr = true;
2169
2170 for_each_intel_crtc(&dev_priv->drm, crtc) {
2171 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2172
2173 if (!crtc->active)
2174 continue;
2175
2176 if (!wm_state->cxsr)
2177 wm->cxsr = false;
2178
2179 num_active_pipes++;
2180 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2181 }
2182
2183 if (num_active_pipes != 1)
2184 wm->cxsr = false;
2185
2186 if (num_active_pipes > 1)
2187 wm->level = VLV_WM_LEVEL_PM2;
2188
2189 for_each_intel_crtc(&dev_priv->drm, crtc) {
2190 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2191 enum pipe pipe = crtc->pipe;
2192
2193 wm->pipe[pipe] = wm_state->wm[wm->level];
2194 if (crtc->active && wm->cxsr)
2195 wm->sr = wm_state->sr[wm->level];
2196
2197 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2198 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2199 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2200 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2201 }
2202}
2203
2204static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2205{
2206 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2207 struct vlv_wm_values new_wm = {};
2208
2209 vlv_merge_wm(dev_priv, &new_wm);
2210
2211 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2212 return;
2213
2214 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2215 chv_set_memory_dvfs(dev_priv, false);
2216
2217 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2218 chv_set_memory_pm5(dev_priv, false);
2219
2220 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2221 _intel_set_memory_cxsr(dev_priv, false);
2222
2223 vlv_write_wm_values(dev_priv, &new_wm);
2224
2225 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2226 _intel_set_memory_cxsr(dev_priv, true);
2227
2228 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2229 chv_set_memory_pm5(dev_priv, true);
2230
2231 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2232 chv_set_memory_dvfs(dev_priv, true);
2233
2234 *old_wm = new_wm;
2235}
2236
2237static void vlv_initial_watermarks(struct intel_atomic_state *state,
2238 struct intel_crtc *crtc)
2239{
2240 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2241 const struct intel_crtc_state *crtc_state =
2242 intel_atomic_get_new_crtc_state(state, crtc);
2243
2244 mutex_lock(&dev_priv->wm.wm_mutex);
2245 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2246 vlv_program_watermarks(dev_priv);
2247 mutex_unlock(&dev_priv->wm.wm_mutex);
2248}
2249
2250static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2251 struct intel_crtc *crtc)
2252{
2253 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2254 const struct intel_crtc_state *crtc_state =
2255 intel_atomic_get_new_crtc_state(state, crtc);
2256
2257 if (!crtc_state->wm.need_postvbl_update)
2258 return;
2259
2260 mutex_lock(&dev_priv->wm.wm_mutex);
2261 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2262 vlv_program_watermarks(dev_priv);
2263 mutex_unlock(&dev_priv->wm.wm_mutex);
2264}
2265
2266static void i965_update_wm(struct intel_crtc *unused_crtc)
2267{
2268 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2269 struct intel_crtc *crtc;
2270 int srwm = 1;
2271 int cursor_sr = 16;
2272 bool cxsr_enabled;
2273
2274 /* Calc sr entries for one plane configs */
2275 crtc = single_enabled_crtc(dev_priv);
2276 if (crtc) {
2277 /* self-refresh has much higher latency */
2278 static const int sr_latency_ns = 12000;
2279 const struct drm_display_mode *pipe_mode =
2280 &crtc->config->hw.pipe_mode;
2281 const struct drm_framebuffer *fb =
2282 crtc->base.primary->state->fb;
2283 int clock = pipe_mode->crtc_clock;
2284 int htotal = pipe_mode->crtc_htotal;
2285 int hdisplay = crtc->config->pipe_src_w;
2286 int cpp = fb->format->cpp[0];
2287 int entries;
2288
2289 entries = intel_wm_method2(clock, htotal,
2290 hdisplay, cpp, sr_latency_ns / 100);
2291 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2292 srwm = I965_FIFO_SIZE - entries;
2293 if (srwm < 0)
2294 srwm = 1;
2295 srwm &= 0x1ff;
2296 drm_dbg_kms(&dev_priv->drm,
2297 "self-refresh entries: %d, wm: %d\n",
2298 entries, srwm);
2299
2300 entries = intel_wm_method2(clock, htotal,
2301 crtc->base.cursor->state->crtc_w, 4,
2302 sr_latency_ns / 100);
2303 entries = DIV_ROUND_UP(entries,
2304 i965_cursor_wm_info.cacheline_size) +
2305 i965_cursor_wm_info.guard_size;
2306
2307 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2308 if (cursor_sr > i965_cursor_wm_info.max_wm)
2309 cursor_sr = i965_cursor_wm_info.max_wm;
2310
2311 drm_dbg_kms(&dev_priv->drm,
2312 "self-refresh watermark: display plane %d "
2313 "cursor %d\n", srwm, cursor_sr);
2314
2315 cxsr_enabled = true;
2316 } else {
2317 cxsr_enabled = false;
2318 /* Turn off self refresh if both pipes are enabled */
2319 intel_set_memory_cxsr(dev_priv, false);
2320 }
2321
2322 drm_dbg_kms(&dev_priv->drm,
2323 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2324 srwm);
2325
2326 /* 965 has limitations... */
2327 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2328 FW_WM(8, CURSORB) |
2329 FW_WM(8, PLANEB) |
2330 FW_WM(8, PLANEA));
2331 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2332 FW_WM(8, PLANEC_OLD));
2333 /* update cursor SR watermark */
2334 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2335
2336 if (cxsr_enabled)
2337 intel_set_memory_cxsr(dev_priv, true);
2338}
2339
2340#undef FW_WM
2341
2342static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2343{
2344 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2345 const struct intel_watermark_params *wm_info;
2346 u32 fwater_lo;
2347 u32 fwater_hi;
2348 int cwm, srwm = 1;
2349 int fifo_size;
2350 int planea_wm, planeb_wm;
2351 struct intel_crtc *crtc, *enabled = NULL;
2352
2353 if (IS_I945GM(dev_priv))
2354 wm_info = &i945_wm_info;
2355 else if (DISPLAY_VER(dev_priv) != 2)
2356 wm_info = &i915_wm_info;
2357 else
2358 wm_info = &i830_a_wm_info;
2359
2360 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2361 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2362 if (intel_crtc_active(crtc)) {
2363 const struct drm_display_mode *pipe_mode =
2364 &crtc->config->hw.pipe_mode;
2365 const struct drm_framebuffer *fb =
2366 crtc->base.primary->state->fb;
2367 int cpp;
2368
2369 if (DISPLAY_VER(dev_priv) == 2)
2370 cpp = 4;
2371 else
2372 cpp = fb->format->cpp[0];
2373
2374 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2375 wm_info, fifo_size, cpp,
2376 pessimal_latency_ns);
2377 enabled = crtc;
2378 } else {
2379 planea_wm = fifo_size - wm_info->guard_size;
2380 if (planea_wm > (long)wm_info->max_wm)
2381 planea_wm = wm_info->max_wm;
2382 }
2383
2384 if (DISPLAY_VER(dev_priv) == 2)
2385 wm_info = &i830_bc_wm_info;
2386
2387 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2388 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2389 if (intel_crtc_active(crtc)) {
2390 const struct drm_display_mode *pipe_mode =
2391 &crtc->config->hw.pipe_mode;
2392 const struct drm_framebuffer *fb =
2393 crtc->base.primary->state->fb;
2394 int cpp;
2395
2396 if (DISPLAY_VER(dev_priv) == 2)
2397 cpp = 4;
2398 else
2399 cpp = fb->format->cpp[0];
2400
2401 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2402 wm_info, fifo_size, cpp,
2403 pessimal_latency_ns);
2404 if (enabled == NULL)
2405 enabled = crtc;
2406 else
2407 enabled = NULL;
2408 } else {
2409 planeb_wm = fifo_size - wm_info->guard_size;
2410 if (planeb_wm > (long)wm_info->max_wm)
2411 planeb_wm = wm_info->max_wm;
2412 }
2413
2414 drm_dbg_kms(&dev_priv->drm,
2415 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2416
2417 if (IS_I915GM(dev_priv) && enabled) {
2418 struct drm_i915_gem_object *obj;
2419
2420 obj = intel_fb_obj(enabled->base.primary->state->fb);
2421
2422 /* self-refresh seems busted with untiled */
2423 if (!i915_gem_object_is_tiled(obj))
2424 enabled = NULL;
2425 }
2426
2427 /*
2428 * Overlay gets an aggressive default since video jitter is bad.
2429 */
2430 cwm = 2;
2431
2432 /* Play safe and disable self-refresh before adjusting watermarks. */
2433 intel_set_memory_cxsr(dev_priv, false);
2434
2435 /* Calc sr entries for one plane configs */
2436 if (HAS_FW_BLC(dev_priv) && enabled) {
2437 /* self-refresh has much higher latency */
2438 static const int sr_latency_ns = 6000;
2439 const struct drm_display_mode *pipe_mode =
2440 &enabled->config->hw.pipe_mode;
2441 const struct drm_framebuffer *fb =
2442 enabled->base.primary->state->fb;
2443 int clock = pipe_mode->crtc_clock;
2444 int htotal = pipe_mode->crtc_htotal;
2445 int hdisplay = enabled->config->pipe_src_w;
2446 int cpp;
2447 int entries;
2448
2449 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2450 cpp = 4;
2451 else
2452 cpp = fb->format->cpp[0];
2453
2454 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2455 sr_latency_ns / 100);
2456 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2457 drm_dbg_kms(&dev_priv->drm,
2458 "self-refresh entries: %d\n", entries);
2459 srwm = wm_info->fifo_size - entries;
2460 if (srwm < 0)
2461 srwm = 1;
2462
2463 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2464 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2465 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2466 else
2467 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2468 }
2469
2470 drm_dbg_kms(&dev_priv->drm,
2471 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2472 planea_wm, planeb_wm, cwm, srwm);
2473
2474 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2475 fwater_hi = (cwm & 0x1f);
2476
2477 /* Set request length to 8 cachelines per fetch */
2478 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2479 fwater_hi = fwater_hi | (1 << 8);
2480
2481 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2483
2484 if (enabled)
2485 intel_set_memory_cxsr(dev_priv, true);
2486}
2487
2488static void i845_update_wm(struct intel_crtc *unused_crtc)
2489{
2490 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2491 struct intel_crtc *crtc;
2492 const struct drm_display_mode *pipe_mode;
2493 u32 fwater_lo;
2494 int planea_wm;
2495
2496 crtc = single_enabled_crtc(dev_priv);
2497 if (crtc == NULL)
2498 return;
2499
2500 pipe_mode = &crtc->config->hw.pipe_mode;
2501 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
2502 &i845_wm_info,
2503 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2504 4, pessimal_latency_ns);
2505 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2506 fwater_lo |= (3<<8) | planea_wm;
2507
2508 drm_dbg_kms(&dev_priv->drm,
2509 "Setting FIFO watermarks - A: %d\n", planea_wm);
2510
2511 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2512}
2513
2514/* latency must be in 0.1us units. */
2515static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2516 unsigned int cpp,
2517 unsigned int latency)
2518{
2519 unsigned int ret;
2520
2521 ret = intel_wm_method1(pixel_rate, cpp, latency);
2522 ret = DIV_ROUND_UP(ret, 64) + 2;
2523
2524 return ret;
2525}
2526
2527/* latency must be in 0.1us units. */
2528static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2529 unsigned int htotal,
2530 unsigned int width,
2531 unsigned int cpp,
2532 unsigned int latency)
2533{
2534 unsigned int ret;
2535
2536 ret = intel_wm_method2(pixel_rate, htotal,
2537 width, cpp, latency);
2538 ret = DIV_ROUND_UP(ret, 64) + 2;
2539
2540 return ret;
2541}
2542
2543static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2544{
2545 /*
2546 * Neither of these should be possible since this function shouldn't be
2547 * called if the CRTC is off or the plane is invisible. But let's be
2548 * extra paranoid to avoid a potential divide-by-zero if we screw up
2549 * elsewhere in the driver.
2550 */
2551 if (WARN_ON(!cpp))
2552 return 0;
2553 if (WARN_ON(!horiz_pixels))
2554 return 0;
2555
2556 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2557}
2558
2559struct ilk_wm_maximums {
2560 u16 pri;
2561 u16 spr;
2562 u16 cur;
2563 u16 fbc;
2564};
2565
2566/*
2567 * For both WM_PIPE and WM_LP.
2568 * mem_value must be in 0.1us units.
2569 */
2570static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2571 const struct intel_plane_state *plane_state,
2572 u32 mem_value, bool is_lp)
2573{
2574 u32 method1, method2;
2575 int cpp;
2576
2577 if (mem_value == 0)
2578 return U32_MAX;
2579
2580 if (!intel_wm_plane_visible(crtc_state, plane_state))
2581 return 0;
2582
2583 cpp = plane_state->hw.fb->format->cpp[0];
2584
2585 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2586
2587 if (!is_lp)
2588 return method1;
2589
2590 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2591 crtc_state->hw.pipe_mode.crtc_htotal,
2592 drm_rect_width(&plane_state->uapi.dst),
2593 cpp, mem_value);
2594
2595 return min(method1, method2);
2596}
2597
2598/*
2599 * For both WM_PIPE and WM_LP.
2600 * mem_value must be in 0.1us units.
2601 */
2602static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2603 const struct intel_plane_state *plane_state,
2604 u32 mem_value)
2605{
2606 u32 method1, method2;
2607 int cpp;
2608
2609 if (mem_value == 0)
2610 return U32_MAX;
2611
2612 if (!intel_wm_plane_visible(crtc_state, plane_state))
2613 return 0;
2614
2615 cpp = plane_state->hw.fb->format->cpp[0];
2616
2617 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2618 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2619 crtc_state->hw.pipe_mode.crtc_htotal,
2620 drm_rect_width(&plane_state->uapi.dst),
2621 cpp, mem_value);
2622 return min(method1, method2);
2623}
2624
2625/*
2626 * For both WM_PIPE and WM_LP.
2627 * mem_value must be in 0.1us units.
2628 */
2629static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2630 const struct intel_plane_state *plane_state,
2631 u32 mem_value)
2632{
2633 int cpp;
2634
2635 if (mem_value == 0)
2636 return U32_MAX;
2637
2638 if (!intel_wm_plane_visible(crtc_state, plane_state))
2639 return 0;
2640
2641 cpp = plane_state->hw.fb->format->cpp[0];
2642
2643 return ilk_wm_method2(crtc_state->pixel_rate,
2644 crtc_state->hw.pipe_mode.crtc_htotal,
2645 drm_rect_width(&plane_state->uapi.dst),
2646 cpp, mem_value);
2647}
2648
2649/* Only for WM_LP. */
2650static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2651 const struct intel_plane_state *plane_state,
2652 u32 pri_val)
2653{
2654 int cpp;
2655
2656 if (!intel_wm_plane_visible(crtc_state, plane_state))
2657 return 0;
2658
2659 cpp = plane_state->hw.fb->format->cpp[0];
2660
2661 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2662 cpp);
2663}
2664
2665static unsigned int
2666ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2667{
2668 if (DISPLAY_VER(dev_priv) >= 8)
2669 return 3072;
2670 else if (DISPLAY_VER(dev_priv) >= 7)
2671 return 768;
2672 else
2673 return 512;
2674}
2675
2676static unsigned int
2677ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2678 int level, bool is_sprite)
2679{
2680 if (DISPLAY_VER(dev_priv) >= 8)
2681 /* BDW primary/sprite plane watermarks */
2682 return level == 0 ? 255 : 2047;
2683 else if (DISPLAY_VER(dev_priv) >= 7)
2684 /* IVB/HSW primary/sprite plane watermarks */
2685 return level == 0 ? 127 : 1023;
2686 else if (!is_sprite)
2687 /* ILK/SNB primary plane watermarks */
2688 return level == 0 ? 127 : 511;
2689 else
2690 /* ILK/SNB sprite plane watermarks */
2691 return level == 0 ? 63 : 255;
2692}
2693
2694static unsigned int
2695ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2696{
2697 if (DISPLAY_VER(dev_priv) >= 7)
2698 return level == 0 ? 63 : 255;
2699 else
2700 return level == 0 ? 31 : 63;
2701}
2702
2703static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2704{
2705 if (DISPLAY_VER(dev_priv) >= 8)
2706 return 31;
2707 else
2708 return 15;
2709}
2710
2711/* Calculate the maximum primary/sprite plane watermark */
2712static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2713 int level,
2714 const struct intel_wm_config *config,
2715 enum intel_ddb_partitioning ddb_partitioning,
2716 bool is_sprite)
2717{
2718 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2719
2720 /* if sprites aren't enabled, sprites get nothing */
2721 if (is_sprite && !config->sprites_enabled)
2722 return 0;
2723
2724 /* HSW allows LP1+ watermarks even with multiple pipes */
2725 if (level == 0 || config->num_pipes_active > 1) {
2726 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2727
2728 /*
2729 * For some reason the non self refresh
2730 * FIFO size is only half of the self
2731 * refresh FIFO size on ILK/SNB.
2732 */
2733 if (DISPLAY_VER(dev_priv) <= 6)
2734 fifo_size /= 2;
2735 }
2736
2737 if (config->sprites_enabled) {
2738 /* level 0 is always calculated with 1:1 split */
2739 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2740 if (is_sprite)
2741 fifo_size *= 5;
2742 fifo_size /= 6;
2743 } else {
2744 fifo_size /= 2;
2745 }
2746 }
2747
2748 /* clamp to max that the registers can hold */
2749 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2750}
2751
2752/* Calculate the maximum cursor plane watermark */
2753static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2754 int level,
2755 const struct intel_wm_config *config)
2756{
2757 /* HSW LP1+ watermarks w/ multiple pipes */
2758 if (level > 0 && config->num_pipes_active > 1)
2759 return 64;
2760
2761 /* otherwise just report max that registers can hold */
2762 return ilk_cursor_wm_reg_max(dev_priv, level);
2763}
2764
2765static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2766 int level,
2767 const struct intel_wm_config *config,
2768 enum intel_ddb_partitioning ddb_partitioning,
2769 struct ilk_wm_maximums *max)
2770{
2771 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2772 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2773 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2774 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2775}
2776
2777static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2778 int level,
2779 struct ilk_wm_maximums *max)
2780{
2781 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2782 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2783 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2784 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2785}
2786
2787static bool ilk_validate_wm_level(int level,
2788 const struct ilk_wm_maximums *max,
2789 struct intel_wm_level *result)
2790{
2791 bool ret;
2792
2793 /* already determined to be invalid? */
2794 if (!result->enable)
2795 return false;
2796
2797 result->enable = result->pri_val <= max->pri &&
2798 result->spr_val <= max->spr &&
2799 result->cur_val <= max->cur;
2800
2801 ret = result->enable;
2802
2803 /*
2804 * HACK until we can pre-compute everything,
2805 * and thus fail gracefully if LP0 watermarks
2806 * are exceeded...
2807 */
2808 if (level == 0 && !result->enable) {
2809 if (result->pri_val > max->pri)
2810 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2811 level, result->pri_val, max->pri);
2812 if (result->spr_val > max->spr)
2813 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2814 level, result->spr_val, max->spr);
2815 if (result->cur_val > max->cur)
2816 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2817 level, result->cur_val, max->cur);
2818
2819 result->pri_val = min_t(u32, result->pri_val, max->pri);
2820 result->spr_val = min_t(u32, result->spr_val, max->spr);
2821 result->cur_val = min_t(u32, result->cur_val, max->cur);
2822 result->enable = true;
2823 }
2824
2825 return ret;
2826}
2827
2828static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2829 const struct intel_crtc *crtc,
2830 int level,
2831 struct intel_crtc_state *crtc_state,
2832 const struct intel_plane_state *pristate,
2833 const struct intel_plane_state *sprstate,
2834 const struct intel_plane_state *curstate,
2835 struct intel_wm_level *result)
2836{
2837 u16 pri_latency = dev_priv->wm.pri_latency[level];
2838 u16 spr_latency = dev_priv->wm.spr_latency[level];
2839 u16 cur_latency = dev_priv->wm.cur_latency[level];
2840
2841 /* WM1+ latency values stored in 0.5us units */
2842 if (level > 0) {
2843 pri_latency *= 5;
2844 spr_latency *= 5;
2845 cur_latency *= 5;
2846 }
2847
2848 if (pristate) {
2849 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2850 pri_latency, level);
2851 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2852 }
2853
2854 if (sprstate)
2855 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2856
2857 if (curstate)
2858 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2859
2860 result->enable = true;
2861}
2862
2863static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2864 u16 wm[8])
2865{
2866 struct intel_uncore *uncore = &dev_priv->uncore;
2867
2868 if (DISPLAY_VER(dev_priv) >= 9) {
2869 u32 val;
2870 int ret, i;
2871 int level, max_level = ilk_wm_max_level(dev_priv);
2872
2873 /* read the first set of memory latencies[0:3] */
2874 val = 0; /* data0 to be programmed to 0 for first set */
2875 ret = sandybridge_pcode_read(dev_priv,
2876 GEN9_PCODE_READ_MEM_LATENCY,
2877 &val, NULL);
2878
2879 if (ret) {
2880 drm_err(&dev_priv->drm,
2881 "SKL Mailbox read error = %d\n", ret);
2882 return;
2883 }
2884
2885 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2886 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2887 GEN9_MEM_LATENCY_LEVEL_MASK;
2888 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2889 GEN9_MEM_LATENCY_LEVEL_MASK;
2890 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2891 GEN9_MEM_LATENCY_LEVEL_MASK;
2892
2893 /* read the second set of memory latencies[4:7] */
2894 val = 1; /* data0 to be programmed to 1 for second set */
2895 ret = sandybridge_pcode_read(dev_priv,
2896 GEN9_PCODE_READ_MEM_LATENCY,
2897 &val, NULL);
2898 if (ret) {
2899 drm_err(&dev_priv->drm,
2900 "SKL Mailbox read error = %d\n", ret);
2901 return;
2902 }
2903
2904 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2905 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2906 GEN9_MEM_LATENCY_LEVEL_MASK;
2907 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2908 GEN9_MEM_LATENCY_LEVEL_MASK;
2909 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2910 GEN9_MEM_LATENCY_LEVEL_MASK;
2911
2912 /*
2913 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2914 * need to be disabled. We make sure to sanitize the values out
2915 * of the punit to satisfy this requirement.
2916 */
2917 for (level = 1; level <= max_level; level++) {
2918 if (wm[level] == 0) {
2919 for (i = level + 1; i <= max_level; i++)
2920 wm[i] = 0;
2921 break;
2922 }
2923 }
2924
2925 /*
2926 * WaWmMemoryReadLatency:skl+,glk
2927 *
2928 * punit doesn't take into account the read latency so we need
2929 * to add 2us to the various latency levels we retrieve from the
2930 * punit when level 0 response data us 0us.
2931 */
2932 if (wm[0] == 0) {
2933 wm[0] += 2;
2934 for (level = 1; level <= max_level; level++) {
2935 if (wm[level] == 0)
2936 break;
2937 wm[level] += 2;
2938 }
2939 }
2940
2941 /*
2942 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2943 * If we could not get dimm info enable this WA to prevent from
2944 * any underrun. If not able to get Dimm info assume 16GB dimm
2945 * to avoid any underrun.
2946 */
2947 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2948 wm[0] += 1;
2949
2950 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2951 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2952
2953 wm[0] = (sskpd >> 56) & 0xFF;
2954 if (wm[0] == 0)
2955 wm[0] = sskpd & 0xF;
2956 wm[1] = (sskpd >> 4) & 0xFF;
2957 wm[2] = (sskpd >> 12) & 0xFF;
2958 wm[3] = (sskpd >> 20) & 0x1FF;
2959 wm[4] = (sskpd >> 32) & 0x1FF;
2960 } else if (DISPLAY_VER(dev_priv) >= 6) {
2961 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2962
2963 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2964 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2965 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2966 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2967 } else if (DISPLAY_VER(dev_priv) >= 5) {
2968 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2969
2970 /* ILK primary LP0 latency is 700 ns */
2971 wm[0] = 7;
2972 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2973 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2974 } else {
2975 MISSING_CASE(INTEL_DEVID(dev_priv));
2976 }
2977}
2978
2979static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2980 u16 wm[5])
2981{
2982 /* ILK sprite LP0 latency is 1300 ns */
2983 if (DISPLAY_VER(dev_priv) == 5)
2984 wm[0] = 13;
2985}
2986
2987static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2988 u16 wm[5])
2989{
2990 /* ILK cursor LP0 latency is 1300 ns */
2991 if (DISPLAY_VER(dev_priv) == 5)
2992 wm[0] = 13;
2993}
2994
2995int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2996{
2997 /* how many WM levels are we expecting */
2998 if (HAS_HW_SAGV_WM(dev_priv))
2999 return 5;
3000 else if (DISPLAY_VER(dev_priv) >= 9)
3001 return 7;
3002 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3003 return 4;
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3005 return 3;
3006 else
3007 return 2;
3008}
3009
3010static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3011 const char *name,
3012 const u16 wm[])
3013{
3014 int level, max_level = ilk_wm_max_level(dev_priv);
3015
3016 for (level = 0; level <= max_level; level++) {
3017 unsigned int latency = wm[level];
3018
3019 if (latency == 0) {
3020 drm_dbg_kms(&dev_priv->drm,
3021 "%s WM%d latency not provided\n",
3022 name, level);
3023 continue;
3024 }
3025
3026 /*
3027 * - latencies are in us on gen9.
3028 * - before then, WM1+ latency values are in 0.5us units
3029 */
3030 if (DISPLAY_VER(dev_priv) >= 9)
3031 latency *= 10;
3032 else if (level > 0)
3033 latency *= 5;
3034
3035 drm_dbg_kms(&dev_priv->drm,
3036 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3037 wm[level], latency / 10, latency % 10);
3038 }
3039}
3040
3041static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3042 u16 wm[5], u16 min)
3043{
3044 int level, max_level = ilk_wm_max_level(dev_priv);
3045
3046 if (wm[0] >= min)
3047 return false;
3048
3049 wm[0] = max(wm[0], min);
3050 for (level = 1; level <= max_level; level++)
3051 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3052
3053 return true;
3054}
3055
3056static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3057{
3058 bool changed;
3059
3060 /*
3061 * The BIOS provided WM memory latency values are often
3062 * inadequate for high resolution displays. Adjust them.
3063 */
3064 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3065 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3066 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3067
3068 if (!changed)
3069 return;
3070
3071 drm_dbg_kms(&dev_priv->drm,
3072 "WM latency values increased to avoid potential underruns\n");
3073 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3074 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3075 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3076}
3077
3078static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3079{
3080 /*
3081 * On some SNB machines (Thinkpad X220 Tablet at least)
3082 * LP3 usage can cause vblank interrupts to be lost.
3083 * The DEIIR bit will go high but it looks like the CPU
3084 * never gets interrupted.
3085 *
3086 * It's not clear whether other interrupt source could
3087 * be affected or if this is somehow limited to vblank
3088 * interrupts only. To play it safe we disable LP3
3089 * watermarks entirely.
3090 */
3091 if (dev_priv->wm.pri_latency[3] == 0 &&
3092 dev_priv->wm.spr_latency[3] == 0 &&
3093 dev_priv->wm.cur_latency[3] == 0)
3094 return;
3095
3096 dev_priv->wm.pri_latency[3] = 0;
3097 dev_priv->wm.spr_latency[3] = 0;
3098 dev_priv->wm.cur_latency[3] = 0;
3099
3100 drm_dbg_kms(&dev_priv->drm,
3101 "LP3 watermarks disabled due to potential for lost interrupts\n");
3102 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3103 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3104 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3105}
3106
3107static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3108{
3109 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3110
3111 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3112 sizeof(dev_priv->wm.pri_latency));
3113 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3114 sizeof(dev_priv->wm.pri_latency));
3115
3116 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3117 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3118
3119 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3120 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3121 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3122
3123 if (DISPLAY_VER(dev_priv) == 6) {
3124 snb_wm_latency_quirk(dev_priv);
3125 snb_wm_lp3_irq_quirk(dev_priv);
3126 }
3127}
3128
3129static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3130{
3131 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3132 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3133}
3134
3135static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3136 struct intel_pipe_wm *pipe_wm)
3137{
3138 /* LP0 watermark maximums depend on this pipe alone */
3139 const struct intel_wm_config config = {
3140 .num_pipes_active = 1,
3141 .sprites_enabled = pipe_wm->sprites_enabled,
3142 .sprites_scaled = pipe_wm->sprites_scaled,
3143 };
3144 struct ilk_wm_maximums max;
3145
3146 /* LP0 watermarks always use 1/2 DDB partitioning */
3147 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3148
3149 /* At least LP0 must be valid */
3150 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3151 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3152 return false;
3153 }
3154
3155 return true;
3156}
3157
3158/* Compute new watermarks for the pipe */
3159static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3160{
3161 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3162 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3163 struct intel_pipe_wm *pipe_wm;
3164 struct intel_plane *plane;
3165 const struct intel_plane_state *plane_state;
3166 const struct intel_plane_state *pristate = NULL;
3167 const struct intel_plane_state *sprstate = NULL;
3168 const struct intel_plane_state *curstate = NULL;
3169 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3170 struct ilk_wm_maximums max;
3171
3172 pipe_wm = &crtc_state->wm.ilk.optimal;
3173
3174 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3175 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3176 pristate = plane_state;
3177 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3178 sprstate = plane_state;
3179 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3180 curstate = plane_state;
3181 }
3182
3183 pipe_wm->pipe_enabled = crtc_state->hw.active;
3184 if (sprstate) {
3185 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3186 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3187 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3188 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3189 }
3190
3191 usable_level = max_level;
3192
3193 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3194 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3195 usable_level = 1;
3196
3197 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3198 if (pipe_wm->sprites_scaled)
3199 usable_level = 0;
3200
3201 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3202 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3203 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3204
3205 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3206 return -EINVAL;
3207
3208 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3209
3210 for (level = 1; level <= usable_level; level++) {
3211 struct intel_wm_level *wm = &pipe_wm->wm[level];
3212
3213 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3214 pristate, sprstate, curstate, wm);
3215
3216 /*
3217 * Disable any watermark level that exceeds the
3218 * register maximums since such watermarks are
3219 * always invalid.
3220 */
3221 if (!ilk_validate_wm_level(level, &max, wm)) {
3222 memset(wm, 0, sizeof(*wm));
3223 break;
3224 }
3225 }
3226
3227 return 0;
3228}
3229
3230/*
3231 * Build a set of 'intermediate' watermark values that satisfy both the old
3232 * state and the new state. These can be programmed to the hardware
3233 * immediately.
3234 */
3235static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3236{
3237 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3238 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3239 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3240 struct intel_atomic_state *intel_state =
3241 to_intel_atomic_state(newstate->uapi.state);
3242 const struct intel_crtc_state *oldstate =
3243 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3244 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3245 int level, max_level = ilk_wm_max_level(dev_priv);
3246
3247 /*
3248 * Start with the final, target watermarks, then combine with the
3249 * currently active watermarks to get values that are safe both before
3250 * and after the vblank.
3251 */
3252 *a = newstate->wm.ilk.optimal;
3253 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3254 intel_state->skip_intermediate_wm)
3255 return 0;
3256
3257 a->pipe_enabled |= b->pipe_enabled;
3258 a->sprites_enabled |= b->sprites_enabled;
3259 a->sprites_scaled |= b->sprites_scaled;
3260
3261 for (level = 0; level <= max_level; level++) {
3262 struct intel_wm_level *a_wm = &a->wm[level];
3263 const struct intel_wm_level *b_wm = &b->wm[level];
3264
3265 a_wm->enable &= b_wm->enable;
3266 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3267 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3268 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3269 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3270 }
3271
3272 /*
3273 * We need to make sure that these merged watermark values are
3274 * actually a valid configuration themselves. If they're not,
3275 * there's no safe way to transition from the old state to
3276 * the new state, so we need to fail the atomic transaction.
3277 */
3278 if (!ilk_validate_pipe_wm(dev_priv, a))
3279 return -EINVAL;
3280
3281 /*
3282 * If our intermediate WM are identical to the final WM, then we can
3283 * omit the post-vblank programming; only update if it's different.
3284 */
3285 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3286 newstate->wm.need_postvbl_update = true;
3287
3288 return 0;
3289}
3290
3291/*
3292 * Merge the watermarks from all active pipes for a specific level.
3293 */
3294static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3295 int level,
3296 struct intel_wm_level *ret_wm)
3297{
3298 const struct intel_crtc *intel_crtc;
3299
3300 ret_wm->enable = true;
3301
3302 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3303 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3304 const struct intel_wm_level *wm = &active->wm[level];
3305
3306 if (!active->pipe_enabled)
3307 continue;
3308
3309 /*
3310 * The watermark values may have been used in the past,
3311 * so we must maintain them in the registers for some
3312 * time even if the level is now disabled.
3313 */
3314 if (!wm->enable)
3315 ret_wm->enable = false;
3316
3317 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3318 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3319 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3320 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3321 }
3322}
3323
3324/*
3325 * Merge all low power watermarks for all active pipes.
3326 */
3327static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3328 const struct intel_wm_config *config,
3329 const struct ilk_wm_maximums *max,
3330 struct intel_pipe_wm *merged)
3331{
3332 int level, max_level = ilk_wm_max_level(dev_priv);
3333 int last_enabled_level = max_level;
3334
3335 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3336 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3337 config->num_pipes_active > 1)
3338 last_enabled_level = 0;
3339
3340 /* ILK: FBC WM must be disabled always */
3341 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3342
3343 /* merge each WM1+ level */
3344 for (level = 1; level <= max_level; level++) {
3345 struct intel_wm_level *wm = &merged->wm[level];
3346
3347 ilk_merge_wm_level(dev_priv, level, wm);
3348
3349 if (level > last_enabled_level)
3350 wm->enable = false;
3351 else if (!ilk_validate_wm_level(level, max, wm))
3352 /* make sure all following levels get disabled */
3353 last_enabled_level = level - 1;
3354
3355 /*
3356 * The spec says it is preferred to disable
3357 * FBC WMs instead of disabling a WM level.
3358 */
3359 if (wm->fbc_val > max->fbc) {
3360 if (wm->enable)
3361 merged->fbc_wm_enabled = false;
3362 wm->fbc_val = 0;
3363 }
3364 }
3365
3366 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3367 /*
3368 * FIXME this is racy. FBC might get enabled later.
3369 * What we should check here is whether FBC can be
3370 * enabled sometime later.
3371 */
3372 if (DISPLAY_VER(dev_priv) == 5 && !merged->fbc_wm_enabled &&
3373 intel_fbc_is_active(dev_priv)) {
3374 for (level = 2; level <= max_level; level++) {
3375 struct intel_wm_level *wm = &merged->wm[level];
3376
3377 wm->enable = false;
3378 }
3379 }
3380}
3381
3382static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3383{
3384 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3385 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3386}
3387
3388/* The value we need to program into the WM_LPx latency field */
3389static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3390 int level)
3391{
3392 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3393 return 2 * level;
3394 else
3395 return dev_priv->wm.pri_latency[level];
3396}
3397
3398static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3399 const struct intel_pipe_wm *merged,
3400 enum intel_ddb_partitioning partitioning,
3401 struct ilk_wm_values *results)
3402{
3403 struct intel_crtc *intel_crtc;
3404 int level, wm_lp;
3405
3406 results->enable_fbc_wm = merged->fbc_wm_enabled;
3407 results->partitioning = partitioning;
3408
3409 /* LP1+ register values */
3410 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3411 const struct intel_wm_level *r;
3412
3413 level = ilk_wm_lp_to_level(wm_lp, merged);
3414
3415 r = &merged->wm[level];
3416
3417 /*
3418 * Maintain the watermark values even if the level is
3419 * disabled. Doing otherwise could cause underruns.
3420 */
3421 results->wm_lp[wm_lp - 1] =
3422 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3423 (r->pri_val << WM1_LP_SR_SHIFT) |
3424 r->cur_val;
3425
3426 if (r->enable)
3427 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3428
3429 if (DISPLAY_VER(dev_priv) >= 8)
3430 results->wm_lp[wm_lp - 1] |=
3431 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3432 else
3433 results->wm_lp[wm_lp - 1] |=
3434 r->fbc_val << WM1_LP_FBC_SHIFT;
3435
3436 /*
3437 * Always set WM1S_LP_EN when spr_val != 0, even if the
3438 * level is disabled. Doing otherwise could cause underruns.
3439 */
3440 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3441 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3442 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3443 } else
3444 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3445 }
3446
3447 /* LP0 register values */
3448 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3449 enum pipe pipe = intel_crtc->pipe;
3450 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3451 const struct intel_wm_level *r = &pipe_wm->wm[0];
3452
3453 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3454 continue;
3455
3456 results->wm_pipe[pipe] =
3457 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3458 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3459 r->cur_val;
3460 }
3461}
3462
3463/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3464 * case both are at the same level. Prefer r1 in case they're the same. */
3465static struct intel_pipe_wm *
3466ilk_find_best_result(struct drm_i915_private *dev_priv,
3467 struct intel_pipe_wm *r1,
3468 struct intel_pipe_wm *r2)
3469{
3470 int level, max_level = ilk_wm_max_level(dev_priv);
3471 int level1 = 0, level2 = 0;
3472
3473 for (level = 1; level <= max_level; level++) {
3474 if (r1->wm[level].enable)
3475 level1 = level;
3476 if (r2->wm[level].enable)
3477 level2 = level;
3478 }
3479
3480 if (level1 == level2) {
3481 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3482 return r2;
3483 else
3484 return r1;
3485 } else if (level1 > level2) {
3486 return r1;
3487 } else {
3488 return r2;
3489 }
3490}
3491
3492/* dirty bits used to track which watermarks need changes */
3493#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3494#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3495#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3496#define WM_DIRTY_FBC (1 << 24)
3497#define WM_DIRTY_DDB (1 << 25)
3498
3499static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3500 const struct ilk_wm_values *old,
3501 const struct ilk_wm_values *new)
3502{
3503 unsigned int dirty = 0;
3504 enum pipe pipe;
3505 int wm_lp;
3506
3507 for_each_pipe(dev_priv, pipe) {
3508 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3509 dirty |= WM_DIRTY_PIPE(pipe);
3510 /* Must disable LP1+ watermarks too */
3511 dirty |= WM_DIRTY_LP_ALL;
3512 }
3513 }
3514
3515 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3516 dirty |= WM_DIRTY_FBC;
3517 /* Must disable LP1+ watermarks too */
3518 dirty |= WM_DIRTY_LP_ALL;
3519 }
3520
3521 if (old->partitioning != new->partitioning) {
3522 dirty |= WM_DIRTY_DDB;
3523 /* Must disable LP1+ watermarks too */
3524 dirty |= WM_DIRTY_LP_ALL;
3525 }
3526
3527 /* LP1+ watermarks already deemed dirty, no need to continue */
3528 if (dirty & WM_DIRTY_LP_ALL)
3529 return dirty;
3530
3531 /* Find the lowest numbered LP1+ watermark in need of an update... */
3532 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3533 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3534 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3535 break;
3536 }
3537
3538 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3539 for (; wm_lp <= 3; wm_lp++)
3540 dirty |= WM_DIRTY_LP(wm_lp);
3541
3542 return dirty;
3543}
3544
3545static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3546 unsigned int dirty)
3547{
3548 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3549 bool changed = false;
3550
3551 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3552 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3553 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3554 changed = true;
3555 }
3556 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3557 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3558 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3559 changed = true;
3560 }
3561 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3562 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3563 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3564 changed = true;
3565 }
3566
3567 /*
3568 * Don't touch WM1S_LP_EN here.
3569 * Doing so could cause underruns.
3570 */
3571
3572 return changed;
3573}
3574
3575/*
3576 * The spec says we shouldn't write when we don't need, because every write
3577 * causes WMs to be re-evaluated, expending some power.
3578 */
3579static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3580 struct ilk_wm_values *results)
3581{
3582 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3583 unsigned int dirty;
3584 u32 val;
3585
3586 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3587 if (!dirty)
3588 return;
3589
3590 _ilk_disable_lp_wm(dev_priv, dirty);
3591
3592 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3593 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3594 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3595 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3596 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3597 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3598
3599 if (dirty & WM_DIRTY_DDB) {
3600 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3601 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3602 if (results->partitioning == INTEL_DDB_PART_1_2)
3603 val &= ~WM_MISC_DATA_PARTITION_5_6;
3604 else
3605 val |= WM_MISC_DATA_PARTITION_5_6;
3606 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3607 } else {
3608 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3609 if (results->partitioning == INTEL_DDB_PART_1_2)
3610 val &= ~DISP_DATA_PARTITION_5_6;
3611 else
3612 val |= DISP_DATA_PARTITION_5_6;
3613 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3614 }
3615 }
3616
3617 if (dirty & WM_DIRTY_FBC) {
3618 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3619 if (results->enable_fbc_wm)
3620 val &= ~DISP_FBC_WM_DIS;
3621 else
3622 val |= DISP_FBC_WM_DIS;
3623 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3624 }
3625
3626 if (dirty & WM_DIRTY_LP(1) &&
3627 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3628 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3629
3630 if (DISPLAY_VER(dev_priv) >= 7) {
3631 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3632 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3633 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3634 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3635 }
3636
3637 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3638 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3639 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3640 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3641 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3642 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3643
3644 dev_priv->wm.hw = *results;
3645}
3646
3647bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3648{
3649 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3650}
3651
3652u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3653{
3654 u8 enabled_slices = 0;
3655 enum dbuf_slice slice;
3656
3657 for_each_dbuf_slice(dev_priv, slice) {
3658 if (intel_uncore_read(&dev_priv->uncore,
3659 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3660 enabled_slices |= BIT(slice);
3661 }
3662
3663 return enabled_slices;
3664}
3665
3666/*
3667 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3668 * so assume we'll always need it in order to avoid underruns.
3669 */
3670static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3671{
3672 return DISPLAY_VER(dev_priv) == 9;
3673}
3674
3675static bool
3676intel_has_sagv(struct drm_i915_private *dev_priv)
3677{
3678 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3679 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3680}
3681
3682static void
3683skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3684{
3685 if (DISPLAY_VER(dev_priv) >= 12) {
3686 u32 val = 0;
3687 int ret;
3688
3689 ret = sandybridge_pcode_read(dev_priv,
3690 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3691 &val, NULL);
3692 if (!ret) {
3693 dev_priv->sagv_block_time_us = val;
3694 return;
3695 }
3696
3697 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3698 } else if (DISPLAY_VER(dev_priv) == 11) {
3699 dev_priv->sagv_block_time_us = 10;
3700 return;
3701 } else if (DISPLAY_VER(dev_priv) == 10) {
3702 dev_priv->sagv_block_time_us = 20;
3703 return;
3704 } else if (DISPLAY_VER(dev_priv) == 9) {
3705 dev_priv->sagv_block_time_us = 30;
3706 return;
3707 } else {
3708 MISSING_CASE(DISPLAY_VER(dev_priv));
3709 }
3710
3711 /* Default to an unusable block time */
3712 dev_priv->sagv_block_time_us = -1;
3713}
3714
3715/*
3716 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3717 * depending on power and performance requirements. The display engine access
3718 * to system memory is blocked during the adjustment time. Because of the
3719 * blocking time, having this enabled can cause full system hangs and/or pipe
3720 * underruns if we don't meet all of the following requirements:
3721 *
3722 * - <= 1 pipe enabled
3723 * - All planes can enable watermarks for latencies >= SAGV engine block time
3724 * - We're not using an interlaced display configuration
3725 */
3726static int
3727intel_enable_sagv(struct drm_i915_private *dev_priv)
3728{
3729 int ret;
3730
3731 if (!intel_has_sagv(dev_priv))
3732 return 0;
3733
3734 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3735 return 0;
3736
3737 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3738 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3739 GEN9_SAGV_ENABLE);
3740
3741 /* We don't need to wait for SAGV when enabling */
3742
3743 /*
3744 * Some skl systems, pre-release machines in particular,
3745 * don't actually have SAGV.
3746 */
3747 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3748 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3749 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3750 return 0;
3751 } else if (ret < 0) {
3752 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3753 return ret;
3754 }
3755
3756 dev_priv->sagv_status = I915_SAGV_ENABLED;
3757 return 0;
3758}
3759
3760static int
3761intel_disable_sagv(struct drm_i915_private *dev_priv)
3762{
3763 int ret;
3764
3765 if (!intel_has_sagv(dev_priv))
3766 return 0;
3767
3768 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3769 return 0;
3770
3771 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3772 /* bspec says to keep retrying for at least 1 ms */
3773 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3774 GEN9_SAGV_DISABLE,
3775 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3776 1);
3777 /*
3778 * Some skl systems, pre-release machines in particular,
3779 * don't actually have SAGV.
3780 */
3781 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3782 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3783 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3784 return 0;
3785 } else if (ret < 0) {
3786 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3787 return ret;
3788 }
3789
3790 dev_priv->sagv_status = I915_SAGV_DISABLED;
3791 return 0;
3792}
3793
3794void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3795{
3796 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3797 const struct intel_bw_state *new_bw_state;
3798 const struct intel_bw_state *old_bw_state;
3799 u32 new_mask = 0;
3800
3801 /*
3802 * Just return if we can't control SAGV or don't have it.
3803 * This is different from situation when we have SAGV but just can't
3804 * afford it due to DBuf limitation - in case if SAGV is completely
3805 * disabled in a BIOS, we are not even allowed to send a PCode request,
3806 * as it will throw an error. So have to check it here.
3807 */
3808 if (!intel_has_sagv(dev_priv))
3809 return;
3810
3811 new_bw_state = intel_atomic_get_new_bw_state(state);
3812 if (!new_bw_state)
3813 return;
3814
3815 if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
3816 intel_disable_sagv(dev_priv);
3817 return;
3818 }
3819
3820 old_bw_state = intel_atomic_get_old_bw_state(state);
3821 /*
3822 * Nothing to mask
3823 */
3824 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3825 return;
3826
3827 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3828
3829 /*
3830 * If new mask is zero - means there is nothing to mask,
3831 * we can only unmask, which should be done in unmask.
3832 */
3833 if (!new_mask)
3834 return;
3835
3836 /*
3837 * Restrict required qgv points before updating the configuration.
3838 * According to BSpec we can't mask and unmask qgv points at the same
3839 * time. Also masking should be done before updating the configuration
3840 * and unmasking afterwards.
3841 */
3842 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3843}
3844
3845void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3846{
3847 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3848 const struct intel_bw_state *new_bw_state;
3849 const struct intel_bw_state *old_bw_state;
3850 u32 new_mask = 0;
3851
3852 /*
3853 * Just return if we can't control SAGV or don't have it.
3854 * This is different from situation when we have SAGV but just can't
3855 * afford it due to DBuf limitation - in case if SAGV is completely
3856 * disabled in a BIOS, we are not even allowed to send a PCode request,
3857 * as it will throw an error. So have to check it here.
3858 */
3859 if (!intel_has_sagv(dev_priv))
3860 return;
3861
3862 new_bw_state = intel_atomic_get_new_bw_state(state);
3863 if (!new_bw_state)
3864 return;
3865
3866 if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
3867 intel_enable_sagv(dev_priv);
3868 return;
3869 }
3870
3871 old_bw_state = intel_atomic_get_old_bw_state(state);
3872 /*
3873 * Nothing to unmask
3874 */
3875 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3876 return;
3877
3878 new_mask = new_bw_state->qgv_points_mask;
3879
3880 /*
3881 * Allow required qgv points after updating the configuration.
3882 * According to BSpec we can't mask and unmask qgv points at the same
3883 * time. Also masking should be done before updating the configuration
3884 * and unmasking afterwards.
3885 */
3886 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3887}
3888
3889static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3890{
3891 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3892 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3893 enum plane_id plane_id;
3894 int max_level = INT_MAX;
3895
3896 if (!intel_has_sagv(dev_priv))
3897 return false;
3898
3899 if (!crtc_state->hw.active)
3900 return true;
3901
3902 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3903 return false;
3904
3905 for_each_plane_id_on_crtc(crtc, plane_id) {
3906 const struct skl_plane_wm *wm =
3907 &crtc_state->wm.skl.optimal.planes[plane_id];
3908 int level;
3909
3910 /* Skip this plane if it's not enabled */
3911 if (!wm->wm[0].enable)
3912 continue;
3913
3914 /* Find the highest enabled wm level for this plane */
3915 for (level = ilk_wm_max_level(dev_priv);
3916 !wm->wm[level].enable; --level)
3917 { }
3918
3919 /* Highest common enabled wm level for all planes */
3920 max_level = min(level, max_level);
3921 }
3922
3923 /* No enabled planes? */
3924 if (max_level == INT_MAX)
3925 return true;
3926
3927 for_each_plane_id_on_crtc(crtc, plane_id) {
3928 const struct skl_plane_wm *wm =
3929 &crtc_state->wm.skl.optimal.planes[plane_id];
3930
3931 /*
3932 * All enabled planes must have enabled a common wm level that
3933 * can tolerate memory latencies higher than sagv_block_time_us
3934 */
3935 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3936 return false;
3937 }
3938
3939 return true;
3940}
3941
3942static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3943{
3944 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3945 enum plane_id plane_id;
3946
3947 if (!crtc_state->hw.active)
3948 return true;
3949
3950 for_each_plane_id_on_crtc(crtc, plane_id) {
3951 const struct skl_plane_wm *wm =
3952 &crtc_state->wm.skl.optimal.planes[plane_id];
3953
3954 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3955 return false;
3956 }
3957
3958 return true;
3959}
3960
3961static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3962{
3963 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3964 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3965
3966 if (DISPLAY_VER(dev_priv) >= 12)
3967 return tgl_crtc_can_enable_sagv(crtc_state);
3968 else
3969 return skl_crtc_can_enable_sagv(crtc_state);
3970}
3971
3972bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3973 const struct intel_bw_state *bw_state)
3974{
3975 if (DISPLAY_VER(dev_priv) < 11 &&
3976 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
3977 return false;
3978
3979 return bw_state->pipe_sagv_reject == 0;
3980}
3981
3982static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3983{
3984 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3985 int ret;
3986 struct intel_crtc *crtc;
3987 struct intel_crtc_state *new_crtc_state;
3988 struct intel_bw_state *new_bw_state = NULL;
3989 const struct intel_bw_state *old_bw_state = NULL;
3990 int i;
3991
3992 for_each_new_intel_crtc_in_state(state, crtc,
3993 new_crtc_state, i) {
3994 new_bw_state = intel_atomic_get_bw_state(state);
3995 if (IS_ERR(new_bw_state))
3996 return PTR_ERR(new_bw_state);
3997
3998 old_bw_state = intel_atomic_get_old_bw_state(state);
3999
4000 if (intel_crtc_can_enable_sagv(new_crtc_state))
4001 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4002 else
4003 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4004 }
4005
4006 if (!new_bw_state)
4007 return 0;
4008
4009 new_bw_state->active_pipes =
4010 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4011
4012 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4013 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4014 if (ret)
4015 return ret;
4016 }
4017
4018 for_each_new_intel_crtc_in_state(state, crtc,
4019 new_crtc_state, i) {
4020 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4021
4022 /*
4023 * We store use_sagv_wm in the crtc state rather than relying on
4024 * that bw state since we have no convenient way to get at the
4025 * latter from the plane commit hooks (especially in the legacy
4026 * cursor case)
4027 */
4028 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4029 DISPLAY_VER(dev_priv) >= 12 &&
4030 intel_can_enable_sagv(dev_priv, new_bw_state);
4031 }
4032
4033 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4034 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4035 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4036 if (ret)
4037 return ret;
4038 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4039 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4040 if (ret)
4041 return ret;
4042 }
4043
4044 return 0;
4045}
4046
4047static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4048{
4049 return INTEL_INFO(dev_priv)->dbuf.size /
4050 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4051}
4052
4053static void
4054skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4055 struct skl_ddb_entry *ddb)
4056{
4057 int slice_size = intel_dbuf_slice_size(dev_priv);
4058
4059 if (!slice_mask) {
4060 ddb->start = 0;
4061 ddb->end = 0;
4062 return;
4063 }
4064
4065 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4066 ddb->end = fls(slice_mask) * slice_size;
4067
4068 WARN_ON(ddb->start >= ddb->end);
4069 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4070}
4071
4072static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4073{
4074 struct skl_ddb_entry ddb;
4075
4076 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4077 slice_mask = BIT(DBUF_S1);
4078 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4079 slice_mask = BIT(DBUF_S3);
4080
4081 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4082
4083 return ddb.start;
4084}
4085
4086u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4087 const struct skl_ddb_entry *entry)
4088{
4089 int slice_size = intel_dbuf_slice_size(dev_priv);
4090 enum dbuf_slice start_slice, end_slice;
4091 u8 slice_mask = 0;
4092
4093 if (!skl_ddb_entry_size(entry))
4094 return 0;
4095
4096 start_slice = entry->start / slice_size;
4097 end_slice = (entry->end - 1) / slice_size;
4098
4099 /*
4100 * Per plane DDB entry can in a really worst case be on multiple slices
4101 * but single entry is anyway contigious.
4102 */
4103 while (start_slice <= end_slice) {
4104 slice_mask |= BIT(start_slice);
4105 start_slice++;
4106 }
4107
4108 return slice_mask;
4109}
4110
4111static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4112{
4113 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4114 int hdisplay, vdisplay;
4115
4116 if (!crtc_state->hw.active)
4117 return 0;
4118
4119 /*
4120 * Watermark/ddb requirement highly depends upon width of the
4121 * framebuffer, So instead of allocating DDB equally among pipes
4122 * distribute DDB based on resolution/width of the display.
4123 */
4124 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4125
4126 return hdisplay;
4127}
4128
4129static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4130 enum pipe for_pipe,
4131 unsigned int *weight_start,
4132 unsigned int *weight_end,
4133 unsigned int *weight_total)
4134{
4135 struct drm_i915_private *dev_priv =
4136 to_i915(dbuf_state->base.state->base.dev);
4137 enum pipe pipe;
4138
4139 *weight_start = 0;
4140 *weight_end = 0;
4141 *weight_total = 0;
4142
4143 for_each_pipe(dev_priv, pipe) {
4144 int weight = dbuf_state->weight[pipe];
4145
4146 /*
4147 * Do not account pipes using other slice sets
4148 * luckily as of current BSpec slice sets do not partially
4149 * intersect(pipes share either same one slice or same slice set
4150 * i.e no partial intersection), so it is enough to check for
4151 * equality for now.
4152 */
4153 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4154 continue;
4155
4156 *weight_total += weight;
4157 if (pipe < for_pipe) {
4158 *weight_start += weight;
4159 *weight_end += weight;
4160 } else if (pipe == for_pipe) {
4161 *weight_end += weight;
4162 }
4163 }
4164}
4165
4166static int
4167skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4168{
4169 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4170 unsigned int weight_total, weight_start, weight_end;
4171 const struct intel_dbuf_state *old_dbuf_state =
4172 intel_atomic_get_old_dbuf_state(state);
4173 struct intel_dbuf_state *new_dbuf_state =
4174 intel_atomic_get_new_dbuf_state(state);
4175 struct intel_crtc_state *crtc_state;
4176 struct skl_ddb_entry ddb_slices;
4177 enum pipe pipe = crtc->pipe;
4178 unsigned int mbus_offset = 0;
4179 u32 ddb_range_size;
4180 u32 dbuf_slice_mask;
4181 u32 start, end;
4182 int ret;
4183
4184 if (new_dbuf_state->weight[pipe] == 0) {
4185 new_dbuf_state->ddb[pipe].start = 0;
4186 new_dbuf_state->ddb[pipe].end = 0;
4187 goto out;
4188 }
4189
4190 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4191
4192 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4193 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4194 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4195
4196 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4197 &weight_start, &weight_end, &weight_total);
4198
4199 start = ddb_range_size * weight_start / weight_total;
4200 end = ddb_range_size * weight_end / weight_total;
4201
4202 new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start;
4203 new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end;
4204out:
4205 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4206 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4207 &new_dbuf_state->ddb[pipe]))
4208 return 0;
4209
4210 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4211 if (ret)
4212 return ret;
4213
4214 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4215 if (IS_ERR(crtc_state))
4216 return PTR_ERR(crtc_state);
4217
4218 /*
4219 * Used for checking overlaps, so we need absolute
4220 * offsets instead of MBUS relative offsets.
4221 */
4222 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4223 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4224
4225 drm_dbg_kms(&dev_priv->drm,
4226 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4227 crtc->base.base.id, crtc->base.name,
4228 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4229 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4230 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4231 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4232
4233 return 0;
4234}
4235
4236static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4237 int width, const struct drm_format_info *format,
4238 u64 modifier, unsigned int rotation,
4239 u32 plane_pixel_rate, struct skl_wm_params *wp,
4240 int color_plane);
4241static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4242 int level,
4243 unsigned int latency,
4244 const struct skl_wm_params *wp,
4245 const struct skl_wm_level *result_prev,
4246 struct skl_wm_level *result /* out */);
4247
4248static unsigned int
4249skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4250 int num_active)
4251{
4252 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4253 int level, max_level = ilk_wm_max_level(dev_priv);
4254 struct skl_wm_level wm = {};
4255 int ret, min_ddb_alloc = 0;
4256 struct skl_wm_params wp;
4257
4258 ret = skl_compute_wm_params(crtc_state, 256,
4259 drm_format_info(DRM_FORMAT_ARGB8888),
4260 DRM_FORMAT_MOD_LINEAR,
4261 DRM_MODE_ROTATE_0,
4262 crtc_state->pixel_rate, &wp, 0);
4263 drm_WARN_ON(&dev_priv->drm, ret);
4264
4265 for (level = 0; level <= max_level; level++) {
4266 unsigned int latency = dev_priv->wm.skl_latency[level];
4267
4268 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
4269 if (wm.min_ddb_alloc == U16_MAX)
4270 break;
4271
4272 min_ddb_alloc = wm.min_ddb_alloc;
4273 }
4274
4275 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4276}
4277
4278static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4279 struct skl_ddb_entry *entry, u32 reg)
4280{
4281 entry->start = reg & DDB_ENTRY_MASK;
4282 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4283
4284 if (entry->end)
4285 entry->end += 1;
4286}
4287
4288static void
4289skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4290 const enum pipe pipe,
4291 const enum plane_id plane_id,
4292 struct skl_ddb_entry *ddb_y,
4293 struct skl_ddb_entry *ddb_uv)
4294{
4295 u32 val, val2;
4296 u32 fourcc = 0;
4297
4298 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4299 if (plane_id == PLANE_CURSOR) {
4300 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4301 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4302 return;
4303 }
4304
4305 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4306
4307 /* No DDB allocated for disabled planes */
4308 if (val & PLANE_CTL_ENABLE)
4309 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4310 val & PLANE_CTL_ORDER_RGBX,
4311 val & PLANE_CTL_ALPHA_MASK);
4312
4313 if (DISPLAY_VER(dev_priv) >= 11) {
4314 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4315 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4316 } else {
4317 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4318 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4319
4320 if (fourcc &&
4321 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4322 swap(val, val2);
4323
4324 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4325 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4326 }
4327}
4328
4329void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4330 struct skl_ddb_entry *ddb_y,
4331 struct skl_ddb_entry *ddb_uv)
4332{
4333 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4334 enum intel_display_power_domain power_domain;
4335 enum pipe pipe = crtc->pipe;
4336 intel_wakeref_t wakeref;
4337 enum plane_id plane_id;
4338
4339 power_domain = POWER_DOMAIN_PIPE(pipe);
4340 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4341 if (!wakeref)
4342 return;
4343
4344 for_each_plane_id_on_crtc(crtc, plane_id)
4345 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4346 plane_id,
4347 &ddb_y[plane_id],
4348 &ddb_uv[plane_id]);
4349
4350 intel_display_power_put(dev_priv, power_domain, wakeref);
4351}
4352
4353/*
4354 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4355 * The bspec defines downscale amount as:
4356 *
4357 * """
4358 * Horizontal down scale amount = maximum[1, Horizontal source size /
4359 * Horizontal destination size]
4360 * Vertical down scale amount = maximum[1, Vertical source size /
4361 * Vertical destination size]
4362 * Total down scale amount = Horizontal down scale amount *
4363 * Vertical down scale amount
4364 * """
4365 *
4366 * Return value is provided in 16.16 fixed point form to retain fractional part.
4367 * Caller should take care of dividing & rounding off the value.
4368 */
4369static uint_fixed_16_16_t
4370skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4371 const struct intel_plane_state *plane_state)
4372{
4373 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4374 u32 src_w, src_h, dst_w, dst_h;
4375 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4376 uint_fixed_16_16_t downscale_h, downscale_w;
4377
4378 if (drm_WARN_ON(&dev_priv->drm,
4379 !intel_wm_plane_visible(crtc_state, plane_state)))
4380 return u32_to_fixed16(0);
4381
4382 /*
4383 * Src coordinates are already rotated by 270 degrees for
4384 * the 90/270 degree plane rotation cases (to match the
4385 * GTT mapping), hence no need to account for rotation here.
4386 *
4387 * n.b., src is 16.16 fixed point, dst is whole integer.
4388 */
4389 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4390 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4391 dst_w = drm_rect_width(&plane_state->uapi.dst);
4392 dst_h = drm_rect_height(&plane_state->uapi.dst);
4393
4394 fp_w_ratio = div_fixed16(src_w, dst_w);
4395 fp_h_ratio = div_fixed16(src_h, dst_h);
4396 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4397 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4398
4399 return mul_fixed16(downscale_w, downscale_h);
4400}
4401
4402struct dbuf_slice_conf_entry {
4403 u8 active_pipes;
4404 u8 dbuf_mask[I915_MAX_PIPES];
4405 bool join_mbus;
4406};
4407
4408/*
4409 * Table taken from Bspec 12716
4410 * Pipes do have some preferred DBuf slice affinity,
4411 * plus there are some hardcoded requirements on how
4412 * those should be distributed for multipipe scenarios.
4413 * For more DBuf slices algorithm can get even more messy
4414 * and less readable, so decided to use a table almost
4415 * as is from BSpec itself - that way it is at least easier
4416 * to compare, change and check.
4417 */
4418static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4419/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4420{
4421 {
4422 .active_pipes = BIT(PIPE_A),
4423 .dbuf_mask = {
4424 [PIPE_A] = BIT(DBUF_S1),
4425 },
4426 },
4427 {
4428 .active_pipes = BIT(PIPE_B),
4429 .dbuf_mask = {
4430 [PIPE_B] = BIT(DBUF_S1),
4431 },
4432 },
4433 {
4434 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4435 .dbuf_mask = {
4436 [PIPE_A] = BIT(DBUF_S1),
4437 [PIPE_B] = BIT(DBUF_S2),
4438 },
4439 },
4440 {
4441 .active_pipes = BIT(PIPE_C),
4442 .dbuf_mask = {
4443 [PIPE_C] = BIT(DBUF_S2),
4444 },
4445 },
4446 {
4447 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4448 .dbuf_mask = {
4449 [PIPE_A] = BIT(DBUF_S1),
4450 [PIPE_C] = BIT(DBUF_S2),
4451 },
4452 },
4453 {
4454 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4455 .dbuf_mask = {
4456 [PIPE_B] = BIT(DBUF_S1),
4457 [PIPE_C] = BIT(DBUF_S2),
4458 },
4459 },
4460 {
4461 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4462 .dbuf_mask = {
4463 [PIPE_A] = BIT(DBUF_S1),
4464 [PIPE_B] = BIT(DBUF_S1),
4465 [PIPE_C] = BIT(DBUF_S2),
4466 },
4467 },
4468 {}
4469};
4470
4471/*
4472 * Table taken from Bspec 49255
4473 * Pipes do have some preferred DBuf slice affinity,
4474 * plus there are some hardcoded requirements on how
4475 * those should be distributed for multipipe scenarios.
4476 * For more DBuf slices algorithm can get even more messy
4477 * and less readable, so decided to use a table almost
4478 * as is from BSpec itself - that way it is at least easier
4479 * to compare, change and check.
4480 */
4481static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4482/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4483{
4484 {
4485 .active_pipes = BIT(PIPE_A),
4486 .dbuf_mask = {
4487 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4488 },
4489 },
4490 {
4491 .active_pipes = BIT(PIPE_B),
4492 .dbuf_mask = {
4493 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4494 },
4495 },
4496 {
4497 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4498 .dbuf_mask = {
4499 [PIPE_A] = BIT(DBUF_S2),
4500 [PIPE_B] = BIT(DBUF_S1),
4501 },
4502 },
4503 {
4504 .active_pipes = BIT(PIPE_C),
4505 .dbuf_mask = {
4506 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4507 },
4508 },
4509 {
4510 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4511 .dbuf_mask = {
4512 [PIPE_A] = BIT(DBUF_S1),
4513 [PIPE_C] = BIT(DBUF_S2),
4514 },
4515 },
4516 {
4517 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4518 .dbuf_mask = {
4519 [PIPE_B] = BIT(DBUF_S1),
4520 [PIPE_C] = BIT(DBUF_S2),
4521 },
4522 },
4523 {
4524 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4525 .dbuf_mask = {
4526 [PIPE_A] = BIT(DBUF_S1),
4527 [PIPE_B] = BIT(DBUF_S1),
4528 [PIPE_C] = BIT(DBUF_S2),
4529 },
4530 },
4531 {
4532 .active_pipes = BIT(PIPE_D),
4533 .dbuf_mask = {
4534 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4535 },
4536 },
4537 {
4538 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4539 .dbuf_mask = {
4540 [PIPE_A] = BIT(DBUF_S1),
4541 [PIPE_D] = BIT(DBUF_S2),
4542 },
4543 },
4544 {
4545 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4546 .dbuf_mask = {
4547 [PIPE_B] = BIT(DBUF_S1),
4548 [PIPE_D] = BIT(DBUF_S2),
4549 },
4550 },
4551 {
4552 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4553 .dbuf_mask = {
4554 [PIPE_A] = BIT(DBUF_S1),
4555 [PIPE_B] = BIT(DBUF_S1),
4556 [PIPE_D] = BIT(DBUF_S2),
4557 },
4558 },
4559 {
4560 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4561 .dbuf_mask = {
4562 [PIPE_C] = BIT(DBUF_S1),
4563 [PIPE_D] = BIT(DBUF_S2),
4564 },
4565 },
4566 {
4567 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4568 .dbuf_mask = {
4569 [PIPE_A] = BIT(DBUF_S1),
4570 [PIPE_C] = BIT(DBUF_S2),
4571 [PIPE_D] = BIT(DBUF_S2),
4572 },
4573 },
4574 {
4575 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4576 .dbuf_mask = {
4577 [PIPE_B] = BIT(DBUF_S1),
4578 [PIPE_C] = BIT(DBUF_S2),
4579 [PIPE_D] = BIT(DBUF_S2),
4580 },
4581 },
4582 {
4583 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4584 .dbuf_mask = {
4585 [PIPE_A] = BIT(DBUF_S1),
4586 [PIPE_B] = BIT(DBUF_S1),
4587 [PIPE_C] = BIT(DBUF_S2),
4588 [PIPE_D] = BIT(DBUF_S2),
4589 },
4590 },
4591 {}
4592};
4593
4594static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4595 {
4596 .active_pipes = BIT(PIPE_A),
4597 .dbuf_mask = {
4598 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4599 },
4600 .join_mbus = true,
4601 },
4602 {
4603 .active_pipes = BIT(PIPE_B),
4604 .dbuf_mask = {
4605 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4606 },
4607 .join_mbus = true,
4608 },
4609 {
4610 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4611 .dbuf_mask = {
4612 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4613 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4614 },
4615 },
4616 {
4617 .active_pipes = BIT(PIPE_C),
4618 .dbuf_mask = {
4619 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4620 },
4621 },
4622 {
4623 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4624 .dbuf_mask = {
4625 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4626 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4627 },
4628 },
4629 {
4630 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4631 .dbuf_mask = {
4632 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4633 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4634 },
4635 },
4636 {
4637 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4638 .dbuf_mask = {
4639 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4640 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4641 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4642 },
4643 },
4644 {
4645 .active_pipes = BIT(PIPE_D),
4646 .dbuf_mask = {
4647 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4648 },
4649 },
4650 {
4651 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4652 .dbuf_mask = {
4653 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4654 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4655 },
4656 },
4657 {
4658 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4659 .dbuf_mask = {
4660 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4661 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4662 },
4663 },
4664 {
4665 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4666 .dbuf_mask = {
4667 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4668 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4669 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4670 },
4671 },
4672 {
4673 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4674 .dbuf_mask = {
4675 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4676 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4677 },
4678 },
4679 {
4680 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4681 .dbuf_mask = {
4682 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4683 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4684 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4685 },
4686 },
4687 {
4688 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4689 .dbuf_mask = {
4690 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4691 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4692 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4693 },
4694 },
4695 {
4696 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4697 .dbuf_mask = {
4698 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4699 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4700 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4701 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4702 },
4703 },
4704 {}
4705
4706};
4707
4708static bool check_mbus_joined(u8 active_pipes,
4709 const struct dbuf_slice_conf_entry *dbuf_slices)
4710{
4711 int i;
4712
4713 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4714 if (dbuf_slices[i].active_pipes == active_pipes)
4715 return dbuf_slices[i].join_mbus;
4716 }
4717 return false;
4718}
4719
4720static bool adlp_check_mbus_joined(u8 active_pipes)
4721{
4722 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4723}
4724
4725static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4726 const struct dbuf_slice_conf_entry *dbuf_slices)
4727{
4728 int i;
4729
4730 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
4731 if (dbuf_slices[i].active_pipes == active_pipes)
4732 return dbuf_slices[i].dbuf_mask[pipe];
4733 }
4734 return 0;
4735}
4736
4737/*
4738 * This function finds an entry with same enabled pipe configuration and
4739 * returns correspondent DBuf slice mask as stated in BSpec for particular
4740 * platform.
4741 */
4742static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4743{
4744 /*
4745 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4746 * required calculating "pipe ratio" in order to determine
4747 * if one or two slices can be used for single pipe configurations
4748 * as additional constraint to the existing table.
4749 * However based on recent info, it should be not "pipe ratio"
4750 * but rather ratio between pixel_rate and cdclk with additional
4751 * constants, so for now we are using only table until this is
4752 * clarified. Also this is the reason why crtc_state param is
4753 * still here - we will need it once those additional constraints
4754 * pop up.
4755 */
4756 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
4757}
4758
4759static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
4760{
4761 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
4762}
4763
4764static u32 adlp_compute_dbuf_slices(enum pipe pipe, u32 active_pipes)
4765{
4766 return compute_dbuf_slices(pipe, active_pipes, adlp_allowed_dbufs);
4767}
4768
4769static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
4770{
4771 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4772 enum pipe pipe = crtc->pipe;
4773
4774 if (IS_ALDERLAKE_P(dev_priv))
4775 return adlp_compute_dbuf_slices(pipe, active_pipes);
4776 else if (DISPLAY_VER(dev_priv) == 12)
4777 return tgl_compute_dbuf_slices(pipe, active_pipes);
4778 else if (DISPLAY_VER(dev_priv) == 11)
4779 return icl_compute_dbuf_slices(pipe, active_pipes);
4780 /*
4781 * For anything else just return one slice yet.
4782 * Should be extended for other platforms.
4783 */
4784 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4785}
4786
4787static u64
4788skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4789 const struct intel_plane_state *plane_state,
4790 int color_plane)
4791{
4792 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4793 const struct drm_framebuffer *fb = plane_state->hw.fb;
4794 u32 data_rate;
4795 u32 width = 0, height = 0;
4796 uint_fixed_16_16_t down_scale_amount;
4797 u64 rate;
4798
4799 if (!plane_state->uapi.visible)
4800 return 0;
4801
4802 if (plane->id == PLANE_CURSOR)
4803 return 0;
4804
4805 if (color_plane == 1 &&
4806 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4807 return 0;
4808
4809 /*
4810 * Src coordinates are already rotated by 270 degrees for
4811 * the 90/270 degree plane rotation cases (to match the
4812 * GTT mapping), hence no need to account for rotation here.
4813 */
4814 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4815 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4816
4817 /* UV plane does 1/2 pixel sub-sampling */
4818 if (color_plane == 1) {
4819 width /= 2;
4820 height /= 2;
4821 }
4822
4823 data_rate = width * height;
4824
4825 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4826
4827 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4828
4829 rate *= fb->format->cpp[color_plane];
4830 return rate;
4831}
4832
4833static u64
4834skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4835 struct intel_crtc *crtc)
4836{
4837 struct intel_crtc_state *crtc_state =
4838 intel_atomic_get_new_crtc_state(state, crtc);
4839 const struct intel_plane_state *plane_state;
4840 struct intel_plane *plane;
4841 u64 total_data_rate = 0;
4842 enum plane_id plane_id;
4843 int i;
4844
4845 /* Calculate and cache data rate for each plane */
4846 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4847 if (plane->pipe != crtc->pipe)
4848 continue;
4849
4850 plane_id = plane->id;
4851
4852 /* packed/y */
4853 crtc_state->plane_data_rate[plane_id] =
4854 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4855
4856 /* uv-plane */
4857 crtc_state->uv_plane_data_rate[plane_id] =
4858 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4859 }
4860
4861 for_each_plane_id_on_crtc(crtc, plane_id) {
4862 total_data_rate += crtc_state->plane_data_rate[plane_id];
4863 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
4864 }
4865
4866 return total_data_rate;
4867}
4868
4869static u64
4870icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4871 struct intel_crtc *crtc)
4872{
4873 struct intel_crtc_state *crtc_state =
4874 intel_atomic_get_new_crtc_state(state, crtc);
4875 const struct intel_plane_state *plane_state;
4876 struct intel_plane *plane;
4877 u64 total_data_rate = 0;
4878 enum plane_id plane_id;
4879 int i;
4880
4881 /* Calculate and cache data rate for each plane */
4882 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4883 if (plane->pipe != crtc->pipe)
4884 continue;
4885
4886 plane_id = plane->id;
4887
4888 if (!plane_state->planar_linked_plane) {
4889 crtc_state->plane_data_rate[plane_id] =
4890 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4891 } else {
4892 enum plane_id y_plane_id;
4893
4894 /*
4895 * The slave plane might not iterate in
4896 * intel_atomic_crtc_state_for_each_plane_state(),
4897 * and needs the master plane state which may be
4898 * NULL if we try get_new_plane_state(), so we
4899 * always calculate from the master.
4900 */
4901 if (plane_state->planar_slave)
4902 continue;
4903
4904 /* Y plane rate is calculated on the slave */
4905 y_plane_id = plane_state->planar_linked_plane->id;
4906 crtc_state->plane_data_rate[y_plane_id] =
4907 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4908
4909 crtc_state->plane_data_rate[plane_id] =
4910 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4911 }
4912 }
4913
4914 for_each_plane_id_on_crtc(crtc, plane_id)
4915 total_data_rate += crtc_state->plane_data_rate[plane_id];
4916
4917 return total_data_rate;
4918}
4919
4920const struct skl_wm_level *
4921skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4922 enum plane_id plane_id,
4923 int level)
4924{
4925 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4926
4927 if (level == 0 && pipe_wm->use_sagv_wm)
4928 return &wm->sagv.wm0;
4929
4930 return &wm->wm[level];
4931}
4932
4933const struct skl_wm_level *
4934skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4935 enum plane_id plane_id)
4936{
4937 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4938
4939 if (pipe_wm->use_sagv_wm)
4940 return &wm->sagv.trans_wm;
4941
4942 return &wm->trans_wm;
4943}
4944
4945/*
4946 * We only disable the watermarks for each plane if
4947 * they exceed the ddb allocation of said plane. This
4948 * is done so that we don't end up touching cursor
4949 * watermarks needlessly when some other plane reduces
4950 * our max possible watermark level.
4951 *
4952 * Bspec has this to say about the PLANE_WM enable bit:
4953 * "All the watermarks at this level for all enabled
4954 * planes must be enabled before the level will be used."
4955 * So this is actually safe to do.
4956 */
4957static void
4958skl_check_wm_level(struct skl_wm_level *wm, u64 total)
4959{
4960 if (wm->min_ddb_alloc > total)
4961 memset(wm, 0, sizeof(*wm));
4962}
4963
4964static void
4965skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
4966 u64 total, u64 uv_total)
4967{
4968 if (wm->min_ddb_alloc > total ||
4969 uv_wm->min_ddb_alloc > uv_total) {
4970 memset(wm, 0, sizeof(*wm));
4971 memset(uv_wm, 0, sizeof(*uv_wm));
4972 }
4973}
4974
4975static int
4976skl_allocate_plane_ddb(struct intel_atomic_state *state,
4977 struct intel_crtc *crtc)
4978{
4979 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4980 struct intel_crtc_state *crtc_state =
4981 intel_atomic_get_new_crtc_state(state, crtc);
4982 const struct intel_dbuf_state *dbuf_state =
4983 intel_atomic_get_new_dbuf_state(state);
4984 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
4985 int num_active = hweight8(dbuf_state->active_pipes);
4986 u16 alloc_size, start = 0;
4987 u16 total[I915_MAX_PLANES] = {};
4988 u16 uv_total[I915_MAX_PLANES] = {};
4989 u64 total_data_rate;
4990 enum plane_id plane_id;
4991 u32 blocks;
4992 int level;
4993
4994 /* Clear the partitioning for disabled planes. */
4995 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4996 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4997
4998 if (!crtc_state->hw.active)
4999 return 0;
5000
5001 if (DISPLAY_VER(dev_priv) >= 11)
5002 total_data_rate =
5003 icl_get_total_relative_data_rate(state, crtc);
5004 else
5005 total_data_rate =
5006 skl_get_total_relative_data_rate(state, crtc);
5007
5008 alloc_size = skl_ddb_entry_size(alloc);
5009 if (alloc_size == 0)
5010 return 0;
5011
5012 /* Allocate fixed number of blocks for cursor. */
5013 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5014 alloc_size -= total[PLANE_CURSOR];
5015 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
5016 alloc->end - total[PLANE_CURSOR];
5017 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
5018
5019 if (total_data_rate == 0)
5020 return 0;
5021
5022 /*
5023 * Find the highest watermark level for which we can satisfy the block
5024 * requirement of active planes.
5025 */
5026 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5027 blocks = 0;
5028 for_each_plane_id_on_crtc(crtc, plane_id) {
5029 const struct skl_plane_wm *wm =
5030 &crtc_state->wm.skl.optimal.planes[plane_id];
5031
5032 if (plane_id == PLANE_CURSOR) {
5033 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
5034 drm_WARN_ON(&dev_priv->drm,
5035 wm->wm[level].min_ddb_alloc != U16_MAX);
5036 blocks = U32_MAX;
5037 break;
5038 }
5039 continue;
5040 }
5041
5042 blocks += wm->wm[level].min_ddb_alloc;
5043 blocks += wm->uv_wm[level].min_ddb_alloc;
5044 }
5045
5046 if (blocks <= alloc_size) {
5047 alloc_size -= blocks;
5048 break;
5049 }
5050 }
5051
5052 if (level < 0) {
5053 drm_dbg_kms(&dev_priv->drm,
5054 "Requested display configuration exceeds system DDB limitations");
5055 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5056 blocks, alloc_size);
5057 return -EINVAL;
5058 }
5059
5060 /*
5061 * Grant each plane the blocks it requires at the highest achievable
5062 * watermark level, plus an extra share of the leftover blocks
5063 * proportional to its relative data rate.
5064 */
5065 for_each_plane_id_on_crtc(crtc, plane_id) {
5066 const struct skl_plane_wm *wm =
5067 &crtc_state->wm.skl.optimal.planes[plane_id];
5068 u64 rate;
5069 u16 extra;
5070
5071 if (plane_id == PLANE_CURSOR)
5072 continue;
5073
5074 /*
5075 * We've accounted for all active planes; remaining planes are
5076 * all disabled.
5077 */
5078 if (total_data_rate == 0)
5079 break;
5080
5081 rate = crtc_state->plane_data_rate[plane_id];
5082 extra = min_t(u16, alloc_size,
5083 DIV64_U64_ROUND_UP(alloc_size * rate,
5084 total_data_rate));
5085 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
5086 alloc_size -= extra;
5087 total_data_rate -= rate;
5088
5089 if (total_data_rate == 0)
5090 break;
5091
5092 rate = crtc_state->uv_plane_data_rate[plane_id];
5093 extra = min_t(u16, alloc_size,
5094 DIV64_U64_ROUND_UP(alloc_size * rate,
5095 total_data_rate));
5096 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
5097 alloc_size -= extra;
5098 total_data_rate -= rate;
5099 }
5100 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
5101
5102 /* Set the actual DDB start/end points for each plane */
5103 start = alloc->start;
5104 for_each_plane_id_on_crtc(crtc, plane_id) {
5105 struct skl_ddb_entry *plane_alloc =
5106 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5107 struct skl_ddb_entry *uv_plane_alloc =
5108 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5109
5110 if (plane_id == PLANE_CURSOR)
5111 continue;
5112
5113 /* Gen11+ uses a separate plane for UV watermarks */
5114 drm_WARN_ON(&dev_priv->drm,
5115 DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]);
5116
5117 /* Leave disabled planes at (0,0) */
5118 if (total[plane_id]) {
5119 plane_alloc->start = start;
5120 start += total[plane_id];
5121 plane_alloc->end = start;
5122 }
5123
5124 if (uv_total[plane_id]) {
5125 uv_plane_alloc->start = start;
5126 start += uv_total[plane_id];
5127 uv_plane_alloc->end = start;
5128 }
5129 }
5130
5131 /*
5132 * When we calculated watermark values we didn't know how high
5133 * of a level we'd actually be able to hit, so we just marked
5134 * all levels as "enabled." Go back now and disable the ones
5135 * that aren't actually possible.
5136 */
5137 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5138 for_each_plane_id_on_crtc(crtc, plane_id) {
5139 struct skl_plane_wm *wm =
5140 &crtc_state->wm.skl.optimal.planes[plane_id];
5141
5142 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5143 total[plane_id], uv_total[plane_id]);
5144
5145 /*
5146 * Wa_1408961008:icl, ehl
5147 * Underruns with WM1+ disabled
5148 */
5149 if (DISPLAY_VER(dev_priv) == 11 &&
5150 level == 1 && wm->wm[0].enable) {
5151 wm->wm[level].blocks = wm->wm[0].blocks;
5152 wm->wm[level].lines = wm->wm[0].lines;
5153 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5154 }
5155 }
5156 }
5157
5158 /*
5159 * Go back and disable the transition and SAGV watermarks
5160 * if it turns out we don't have enough DDB blocks for them.
5161 */
5162 for_each_plane_id_on_crtc(crtc, plane_id) {
5163 struct skl_plane_wm *wm =
5164 &crtc_state->wm.skl.optimal.planes[plane_id];
5165
5166 skl_check_wm_level(&wm->trans_wm, total[plane_id]);
5167 skl_check_wm_level(&wm->sagv.wm0, total[plane_id]);
5168 skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]);
5169 }
5170
5171 return 0;
5172}
5173
5174/*
5175 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5176 * for the read latency) and cpp should always be <= 8, so that
5177 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5178 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5179*/
5180static uint_fixed_16_16_t
5181skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5182 u8 cpp, u32 latency, u32 dbuf_block_size)
5183{
5184 u32 wm_intermediate_val;
5185 uint_fixed_16_16_t ret;
5186
5187 if (latency == 0)
5188 return FP_16_16_MAX;
5189
5190 wm_intermediate_val = latency * pixel_rate * cpp;
5191 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5192
5193 if (DISPLAY_VER(dev_priv) >= 10)
5194 ret = add_fixed16_u32(ret, 1);
5195
5196 return ret;
5197}
5198
5199static uint_fixed_16_16_t
5200skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5201 uint_fixed_16_16_t plane_blocks_per_line)
5202{
5203 u32 wm_intermediate_val;
5204 uint_fixed_16_16_t ret;
5205
5206 if (latency == 0)
5207 return FP_16_16_MAX;
5208
5209 wm_intermediate_val = latency * pixel_rate;
5210 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5211 pipe_htotal * 1000);
5212 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5213 return ret;
5214}
5215
5216static uint_fixed_16_16_t
5217intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5218{
5219 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5220 u32 pixel_rate;
5221 u32 crtc_htotal;
5222 uint_fixed_16_16_t linetime_us;
5223
5224 if (!crtc_state->hw.active)
5225 return u32_to_fixed16(0);
5226
5227 pixel_rate = crtc_state->pixel_rate;
5228
5229 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5230 return u32_to_fixed16(0);
5231
5232 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5233 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5234
5235 return linetime_us;
5236}
5237
5238static int
5239skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5240 int width, const struct drm_format_info *format,
5241 u64 modifier, unsigned int rotation,
5242 u32 plane_pixel_rate, struct skl_wm_params *wp,
5243 int color_plane)
5244{
5245 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5246 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5247 u32 interm_pbpl;
5248
5249 /* only planar format has two planes */
5250 if (color_plane == 1 &&
5251 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5252 drm_dbg_kms(&dev_priv->drm,
5253 "Non planar format have single plane\n");
5254 return -EINVAL;
5255 }
5256
5257 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5258 modifier == I915_FORMAT_MOD_Yf_TILED ||
5259 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5260 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5261 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5262 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5263 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5264 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5265
5266 wp->width = width;
5267 if (color_plane == 1 && wp->is_planar)
5268 wp->width /= 2;
5269
5270 wp->cpp = format->cpp[color_plane];
5271 wp->plane_pixel_rate = plane_pixel_rate;
5272
5273 if (DISPLAY_VER(dev_priv) >= 11 &&
5274 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5275 wp->dbuf_block_size = 256;
5276 else
5277 wp->dbuf_block_size = 512;
5278
5279 if (drm_rotation_90_or_270(rotation)) {
5280 switch (wp->cpp) {
5281 case 1:
5282 wp->y_min_scanlines = 16;
5283 break;
5284 case 2:
5285 wp->y_min_scanlines = 8;
5286 break;
5287 case 4:
5288 wp->y_min_scanlines = 4;
5289 break;
5290 default:
5291 MISSING_CASE(wp->cpp);
5292 return -EINVAL;
5293 }
5294 } else {
5295 wp->y_min_scanlines = 4;
5296 }
5297
5298 if (skl_needs_memory_bw_wa(dev_priv))
5299 wp->y_min_scanlines *= 2;
5300
5301 wp->plane_bytes_per_line = wp->width * wp->cpp;
5302 if (wp->y_tiled) {
5303 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5304 wp->y_min_scanlines,
5305 wp->dbuf_block_size);
5306
5307 if (DISPLAY_VER(dev_priv) >= 10)
5308 interm_pbpl++;
5309
5310 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5311 wp->y_min_scanlines);
5312 } else {
5313 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5314 wp->dbuf_block_size);
5315
5316 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5317 interm_pbpl++;
5318
5319 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5320 }
5321
5322 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5323 wp->plane_blocks_per_line);
5324
5325 wp->linetime_us = fixed16_to_u32_round_up(
5326 intel_get_linetime_us(crtc_state));
5327
5328 return 0;
5329}
5330
5331static int
5332skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5333 const struct intel_plane_state *plane_state,
5334 struct skl_wm_params *wp, int color_plane)
5335{
5336 const struct drm_framebuffer *fb = plane_state->hw.fb;
5337 int width;
5338
5339 /*
5340 * Src coordinates are already rotated by 270 degrees for
5341 * the 90/270 degree plane rotation cases (to match the
5342 * GTT mapping), hence no need to account for rotation here.
5343 */
5344 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5345
5346 return skl_compute_wm_params(crtc_state, width,
5347 fb->format, fb->modifier,
5348 plane_state->hw.rotation,
5349 intel_plane_pixel_rate(crtc_state, plane_state),
5350 wp, color_plane);
5351}
5352
5353static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5354{
5355 if (DISPLAY_VER(dev_priv) >= 10)
5356 return true;
5357
5358 /* The number of lines are ignored for the level 0 watermark. */
5359 return level > 0;
5360}
5361
5362static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5363{
5364 if (DISPLAY_VER(dev_priv) >= 13)
5365 return 255;
5366 else
5367 return 31;
5368}
5369
5370static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5371 int level,
5372 unsigned int latency,
5373 const struct skl_wm_params *wp,
5374 const struct skl_wm_level *result_prev,
5375 struct skl_wm_level *result /* out */)
5376{
5377 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5378 uint_fixed_16_16_t method1, method2;
5379 uint_fixed_16_16_t selected_result;
5380 u32 blocks, lines, min_ddb_alloc = 0;
5381
5382 if (latency == 0) {
5383 /* reject it */
5384 result->min_ddb_alloc = U16_MAX;
5385 return;
5386 }
5387
5388 /*
5389 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5390 * Display WA #1141: kbl,cfl
5391 */
5392 if ((IS_KABYLAKE(dev_priv) ||
5393 IS_COFFEELAKE(dev_priv) ||
5394 IS_COMETLAKE(dev_priv)) &&
5395 dev_priv->ipc_enabled)
5396 latency += 4;
5397
5398 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5399 latency += 15;
5400
5401 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5402 wp->cpp, latency, wp->dbuf_block_size);
5403 method2 = skl_wm_method2(wp->plane_pixel_rate,
5404 crtc_state->hw.pipe_mode.crtc_htotal,
5405 latency,
5406 wp->plane_blocks_per_line);
5407
5408 if (wp->y_tiled) {
5409 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5410 } else {
5411 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5412 wp->dbuf_block_size < 1) &&
5413 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5414 selected_result = method2;
5415 } else if (latency >= wp->linetime_us) {
5416 if (DISPLAY_VER(dev_priv) == 9)
5417 selected_result = min_fixed16(method1, method2);
5418 else
5419 selected_result = method2;
5420 } else {
5421 selected_result = method1;
5422 }
5423 }
5424
5425 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5426 lines = div_round_up_fixed16(selected_result,
5427 wp->plane_blocks_per_line);
5428
5429 if (DISPLAY_VER(dev_priv) == 9) {
5430 /* Display WA #1125: skl,bxt,kbl */
5431 if (level == 0 && wp->rc_surface)
5432 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5433
5434 /* Display WA #1126: skl,bxt,kbl */
5435 if (level >= 1 && level <= 7) {
5436 if (wp->y_tiled) {
5437 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5438 lines += wp->y_min_scanlines;
5439 } else {
5440 blocks++;
5441 }
5442
5443 /*
5444 * Make sure result blocks for higher latency levels are
5445 * atleast as high as level below the current level.
5446 * Assumption in DDB algorithm optimization for special
5447 * cases. Also covers Display WA #1125 for RC.
5448 */
5449 if (result_prev->blocks > blocks)
5450 blocks = result_prev->blocks;
5451 }
5452 }
5453
5454 if (DISPLAY_VER(dev_priv) >= 11) {
5455 if (wp->y_tiled) {
5456 int extra_lines;
5457
5458 if (lines % wp->y_min_scanlines == 0)
5459 extra_lines = wp->y_min_scanlines;
5460 else
5461 extra_lines = wp->y_min_scanlines * 2 -
5462 lines % wp->y_min_scanlines;
5463
5464 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5465 wp->plane_blocks_per_line);
5466 } else {
5467 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5468 }
5469 }
5470
5471 if (!skl_wm_has_lines(dev_priv, level))
5472 lines = 0;
5473
5474 if (lines > skl_wm_max_lines(dev_priv)) {
5475 /* reject it */
5476 result->min_ddb_alloc = U16_MAX;
5477 return;
5478 }
5479
5480 /*
5481 * If lines is valid, assume we can use this watermark level
5482 * for now. We'll come back and disable it after we calculate the
5483 * DDB allocation if it turns out we don't actually have enough
5484 * blocks to satisfy it.
5485 */
5486 result->blocks = blocks;
5487 result->lines = lines;
5488 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5489 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5490 result->enable = true;
5491
5492 if (DISPLAY_VER(dev_priv) < 12)
5493 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5494}
5495
5496static void
5497skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5498 const struct skl_wm_params *wm_params,
5499 struct skl_wm_level *levels)
5500{
5501 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5502 int level, max_level = ilk_wm_max_level(dev_priv);
5503 struct skl_wm_level *result_prev = &levels[0];
5504
5505 for (level = 0; level <= max_level; level++) {
5506 struct skl_wm_level *result = &levels[level];
5507 unsigned int latency = dev_priv->wm.skl_latency[level];
5508
5509 skl_compute_plane_wm(crtc_state, level, latency,
5510 wm_params, result_prev, result);
5511
5512 result_prev = result;
5513 }
5514}
5515
5516static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5517 const struct skl_wm_params *wm_params,
5518 struct skl_plane_wm *plane_wm)
5519{
5520 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5521 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5522 struct skl_wm_level *levels = plane_wm->wm;
5523 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5524
5525 skl_compute_plane_wm(crtc_state, 0, latency,
5526 wm_params, &levels[0],
5527 sagv_wm);
5528}
5529
5530static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5531 struct skl_wm_level *trans_wm,
5532 const struct skl_wm_level *wm0,
5533 const struct skl_wm_params *wp)
5534{
5535 u16 trans_min, trans_amount, trans_y_tile_min;
5536 u16 wm0_blocks, trans_offset, blocks;
5537
5538 /* Transition WM don't make any sense if ipc is disabled */
5539 if (!dev_priv->ipc_enabled)
5540 return;
5541
5542 /*
5543 * WaDisableTWM:skl,kbl,cfl,bxt
5544 * Transition WM are not recommended by HW team for GEN9
5545 */
5546 if (DISPLAY_VER(dev_priv) == 9)
5547 return;
5548
5549 if (DISPLAY_VER(dev_priv) >= 11)
5550 trans_min = 4;
5551 else
5552 trans_min = 14;
5553
5554 /* Display WA #1140: glk,cnl */
5555 if (DISPLAY_VER(dev_priv) == 10)
5556 trans_amount = 0;
5557 else
5558 trans_amount = 10; /* This is configurable amount */
5559
5560 trans_offset = trans_min + trans_amount;
5561
5562 /*
5563 * The spec asks for Selected Result Blocks for wm0 (the real value),
5564 * not Result Blocks (the integer value). Pay attention to the capital
5565 * letters. The value wm_l0->blocks is actually Result Blocks, but
5566 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5567 * and since we later will have to get the ceiling of the sum in the
5568 * transition watermarks calculation, we can just pretend Selected
5569 * Result Blocks is Result Blocks minus 1 and it should work for the
5570 * current platforms.
5571 */
5572 wm0_blocks = wm0->blocks - 1;
5573
5574 if (wp->y_tiled) {
5575 trans_y_tile_min =
5576 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5577 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5578 } else {
5579 blocks = wm0_blocks + trans_offset;
5580 }
5581 blocks++;
5582
5583 /*
5584 * Just assume we can enable the transition watermark. After
5585 * computing the DDB we'll come back and disable it if that
5586 * assumption turns out to be false.
5587 */
5588 trans_wm->blocks = blocks;
5589 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5590 trans_wm->enable = true;
5591}
5592
5593static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5594 const struct intel_plane_state *plane_state,
5595 enum plane_id plane_id, int color_plane)
5596{
5597 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5598 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5599 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5600 struct skl_wm_params wm_params;
5601 int ret;
5602
5603 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5604 &wm_params, color_plane);
5605 if (ret)
5606 return ret;
5607
5608 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5609
5610 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5611 &wm->wm[0], &wm_params);
5612
5613 if (DISPLAY_VER(dev_priv) >= 12) {
5614 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5615
5616 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5617 &wm->sagv.wm0, &wm_params);
5618 }
5619
5620 return 0;
5621}
5622
5623static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5624 const struct intel_plane_state *plane_state,
5625 enum plane_id plane_id)
5626{
5627 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5628 struct skl_wm_params wm_params;
5629 int ret;
5630
5631 wm->is_planar = true;
5632
5633 /* uv plane watermarks must also be validated for NV12/Planar */
5634 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5635 &wm_params, 1);
5636 if (ret)
5637 return ret;
5638
5639 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5640
5641 return 0;
5642}
5643
5644static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5645 const struct intel_plane_state *plane_state)
5646{
5647 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5648 enum plane_id plane_id = plane->id;
5649 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5650 const struct drm_framebuffer *fb = plane_state->hw.fb;
5651 int ret;
5652
5653 memset(wm, 0, sizeof(*wm));
5654
5655 if (!intel_wm_plane_visible(crtc_state, plane_state))
5656 return 0;
5657
5658 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5659 plane_id, 0);
5660 if (ret)
5661 return ret;
5662
5663 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5664 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5665 plane_id);
5666 if (ret)
5667 return ret;
5668 }
5669
5670 return 0;
5671}
5672
5673static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5674 const struct intel_plane_state *plane_state)
5675{
5676 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5677 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5678 enum plane_id plane_id = plane->id;
5679 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5680 int ret;
5681
5682 /* Watermarks calculated in master */
5683 if (plane_state->planar_slave)
5684 return 0;
5685
5686 memset(wm, 0, sizeof(*wm));
5687
5688 if (plane_state->planar_linked_plane) {
5689 const struct drm_framebuffer *fb = plane_state->hw.fb;
5690 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5691
5692 drm_WARN_ON(&dev_priv->drm,
5693 !intel_wm_plane_visible(crtc_state, plane_state));
5694 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5695 fb->format->num_planes == 1);
5696
5697 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5698 y_plane_id, 0);
5699 if (ret)
5700 return ret;
5701
5702 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5703 plane_id, 1);
5704 if (ret)
5705 return ret;
5706 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5707 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5708 plane_id, 0);
5709 if (ret)
5710 return ret;
5711 }
5712
5713 return 0;
5714}
5715
5716static int skl_build_pipe_wm(struct intel_atomic_state *state,
5717 struct intel_crtc *crtc)
5718{
5719 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5720 struct intel_crtc_state *crtc_state =
5721 intel_atomic_get_new_crtc_state(state, crtc);
5722 const struct intel_plane_state *plane_state;
5723 struct intel_plane *plane;
5724 int ret, i;
5725
5726 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5727 /*
5728 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5729 * instead but we don't populate that correctly for NV12 Y
5730 * planes so for now hack this.
5731 */
5732 if (plane->pipe != crtc->pipe)
5733 continue;
5734
5735 if (DISPLAY_VER(dev_priv) >= 11)
5736 ret = icl_build_plane_wm(crtc_state, plane_state);
5737 else
5738 ret = skl_build_plane_wm(crtc_state, plane_state);
5739 if (ret)
5740 return ret;
5741 }
5742
5743 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5744
5745 return 0;
5746}
5747
5748static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5749 i915_reg_t reg,
5750 const struct skl_ddb_entry *entry)
5751{
5752 if (entry->end)
5753 intel_de_write_fw(dev_priv, reg,
5754 (entry->end - 1) << 16 | entry->start);
5755 else
5756 intel_de_write_fw(dev_priv, reg, 0);
5757}
5758
5759static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5760 i915_reg_t reg,
5761 const struct skl_wm_level *level)
5762{
5763 u32 val = 0;
5764
5765 if (level->enable)
5766 val |= PLANE_WM_EN;
5767 if (level->ignore_lines)
5768 val |= PLANE_WM_IGNORE_LINES;
5769 val |= level->blocks;
5770 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5771
5772 intel_de_write_fw(dev_priv, reg, val);
5773}
5774
5775void skl_write_plane_wm(struct intel_plane *plane,
5776 const struct intel_crtc_state *crtc_state)
5777{
5778 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5779 int level, max_level = ilk_wm_max_level(dev_priv);
5780 enum plane_id plane_id = plane->id;
5781 enum pipe pipe = plane->pipe;
5782 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5783 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5784 const struct skl_ddb_entry *ddb_y =
5785 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5786 const struct skl_ddb_entry *ddb_uv =
5787 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5788
5789 for (level = 0; level <= max_level; level++)
5790 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5791 skl_plane_wm_level(pipe_wm, plane_id, level));
5792
5793 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5794 skl_plane_trans_wm(pipe_wm, plane_id));
5795
5796 if (HAS_HW_SAGV_WM(dev_priv)) {
5797 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5798 &wm->sagv.wm0);
5799 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5800 &wm->sagv.trans_wm);
5801 }
5802
5803 if (DISPLAY_VER(dev_priv) >= 11) {
5804 skl_ddb_entry_write(dev_priv,
5805 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5806 return;
5807 }
5808
5809 if (wm->is_planar)
5810 swap(ddb_y, ddb_uv);
5811
5812 skl_ddb_entry_write(dev_priv,
5813 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5814 skl_ddb_entry_write(dev_priv,
5815 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5816}
5817
5818void skl_write_cursor_wm(struct intel_plane *plane,
5819 const struct intel_crtc_state *crtc_state)
5820{
5821 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5822 int level, max_level = ilk_wm_max_level(dev_priv);
5823 enum plane_id plane_id = plane->id;
5824 enum pipe pipe = plane->pipe;
5825 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5826 const struct skl_ddb_entry *ddb =
5827 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5828
5829 for (level = 0; level <= max_level; level++)
5830 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5831 skl_plane_wm_level(pipe_wm, plane_id, level));
5832
5833 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5834 skl_plane_trans_wm(pipe_wm, plane_id));
5835
5836 if (HAS_HW_SAGV_WM(dev_priv)) {
5837 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5838
5839 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5840 &wm->sagv.wm0);
5841 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5842 &wm->sagv.trans_wm);
5843 }
5844
5845 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5846}
5847
5848bool skl_wm_level_equals(const struct skl_wm_level *l1,
5849 const struct skl_wm_level *l2)
5850{
5851 return l1->enable == l2->enable &&
5852 l1->ignore_lines == l2->ignore_lines &&
5853 l1->lines == l2->lines &&
5854 l1->blocks == l2->blocks;
5855}
5856
5857static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5858 const struct skl_plane_wm *wm1,
5859 const struct skl_plane_wm *wm2)
5860{
5861 int level, max_level = ilk_wm_max_level(dev_priv);
5862
5863 for (level = 0; level <= max_level; level++) {
5864 /*
5865 * We don't check uv_wm as the hardware doesn't actually
5866 * use it. It only gets used for calculating the required
5867 * ddb allocation.
5868 */
5869 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5870 return false;
5871 }
5872
5873 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5874 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5875 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5876}
5877
5878static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5879 const struct skl_ddb_entry *b)
5880{
5881 return a->start < b->end && b->start < a->end;
5882}
5883
5884static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5885 const struct skl_ddb_entry *b)
5886{
5887 if (a->end && b->end) {
5888 a->start = min(a->start, b->start);
5889 a->end = max(a->end, b->end);
5890 } else if (b->end) {
5891 a->start = b->start;
5892 a->end = b->end;
5893 }
5894}
5895
5896bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5897 const struct skl_ddb_entry *entries,
5898 int num_entries, int ignore_idx)
5899{
5900 int i;
5901
5902 for (i = 0; i < num_entries; i++) {
5903 if (i != ignore_idx &&
5904 skl_ddb_entries_overlap(ddb, &entries[i]))
5905 return true;
5906 }
5907
5908 return false;
5909}
5910
5911static int
5912skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5913 struct intel_crtc_state *new_crtc_state)
5914{
5915 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5916 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5917 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5918 struct intel_plane *plane;
5919
5920 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5921 struct intel_plane_state *plane_state;
5922 enum plane_id plane_id = plane->id;
5923
5924 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5925 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5926 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5927 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5928 continue;
5929
5930 plane_state = intel_atomic_get_plane_state(state, plane);
5931 if (IS_ERR(plane_state))
5932 return PTR_ERR(plane_state);
5933
5934 new_crtc_state->update_planes |= BIT(plane_id);
5935 }
5936
5937 return 0;
5938}
5939
5940static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
5941{
5942 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
5943 u8 enabled_slices;
5944 enum pipe pipe;
5945
5946 /*
5947 * FIXME: For now we always enable slice S1 as per
5948 * the Bspec display initialization sequence.
5949 */
5950 enabled_slices = BIT(DBUF_S1);
5951
5952 for_each_pipe(dev_priv, pipe)
5953 enabled_slices |= dbuf_state->slices[pipe];
5954
5955 return enabled_slices;
5956}
5957
5958static int
5959skl_compute_ddb(struct intel_atomic_state *state)
5960{
5961 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5962 const struct intel_dbuf_state *old_dbuf_state;
5963 struct intel_dbuf_state *new_dbuf_state = NULL;
5964 const struct intel_crtc_state *old_crtc_state;
5965 struct intel_crtc_state *new_crtc_state;
5966 struct intel_crtc *crtc;
5967 int ret, i;
5968
5969 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5970 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5971 if (IS_ERR(new_dbuf_state))
5972 return PTR_ERR(new_dbuf_state);
5973
5974 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5975 break;
5976 }
5977
5978 if (!new_dbuf_state)
5979 return 0;
5980
5981 new_dbuf_state->active_pipes =
5982 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
5983
5984 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
5985 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5986 if (ret)
5987 return ret;
5988 }
5989
5990 for_each_intel_crtc(&dev_priv->drm, crtc) {
5991 enum pipe pipe = crtc->pipe;
5992
5993 new_dbuf_state->slices[pipe] =
5994 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
5995
5996 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
5997 continue;
5998
5999 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6000 if (ret)
6001 return ret;
6002 }
6003
6004 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6005
6006 if (IS_ALDERLAKE_P(dev_priv))
6007 new_dbuf_state->joined_mbus = adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6008
6009 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6010 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6011 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6012 if (ret)
6013 return ret;
6014
6015 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6016 /* TODO: Implement vblank synchronized MBUS joining changes */
6017 ret = intel_modeset_all_pipes(state);
6018 if (ret)
6019 return ret;
6020 }
6021
6022 drm_dbg_kms(&dev_priv->drm,
6023 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6024 old_dbuf_state->enabled_slices,
6025 new_dbuf_state->enabled_slices,
6026 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6027 yesno(old_dbuf_state->joined_mbus),
6028 yesno(new_dbuf_state->joined_mbus));
6029 }
6030
6031 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6032 enum pipe pipe = crtc->pipe;
6033
6034 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6035
6036 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6037 continue;
6038
6039 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6040 if (ret)
6041 return ret;
6042 }
6043
6044 for_each_intel_crtc(&dev_priv->drm, crtc) {
6045 ret = skl_crtc_allocate_ddb(state, crtc);
6046 if (ret)
6047 return ret;
6048 }
6049
6050 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6051 new_crtc_state, i) {
6052 ret = skl_allocate_plane_ddb(state, crtc);
6053 if (ret)
6054 return ret;
6055
6056 ret = skl_ddb_add_affected_planes(old_crtc_state,
6057 new_crtc_state);
6058 if (ret)
6059 return ret;
6060 }
6061
6062 return 0;
6063}
6064
6065static char enast(bool enable)
6066{
6067 return enable ? '*' : ' ';
6068}
6069
6070static void
6071skl_print_wm_changes(struct intel_atomic_state *state)
6072{
6073 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6074 const struct intel_crtc_state *old_crtc_state;
6075 const struct intel_crtc_state *new_crtc_state;
6076 struct intel_plane *plane;
6077 struct intel_crtc *crtc;
6078 int i;
6079
6080 if (!drm_debug_enabled(DRM_UT_KMS))
6081 return;
6082
6083 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6084 new_crtc_state, i) {
6085 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6086
6087 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6088 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6089
6090 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6091 enum plane_id plane_id = plane->id;
6092 const struct skl_ddb_entry *old, *new;
6093
6094 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6095 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6096
6097 if (skl_ddb_entry_equal(old, new))
6098 continue;
6099
6100 drm_dbg_kms(&dev_priv->drm,
6101 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6102 plane->base.base.id, plane->base.name,
6103 old->start, old->end, new->start, new->end,
6104 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6105 }
6106
6107 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6108 enum plane_id plane_id = plane->id;
6109 const struct skl_plane_wm *old_wm, *new_wm;
6110
6111 old_wm = &old_pipe_wm->planes[plane_id];
6112 new_wm = &new_pipe_wm->planes[plane_id];
6113
6114 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6115 continue;
6116
6117 drm_dbg_kms(&dev_priv->drm,
6118 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6119 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6120 plane->base.base.id, plane->base.name,
6121 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6122 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6123 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6124 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6125 enast(old_wm->trans_wm.enable),
6126 enast(old_wm->sagv.wm0.enable),
6127 enast(old_wm->sagv.trans_wm.enable),
6128 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6129 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6130 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6131 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6132 enast(new_wm->trans_wm.enable),
6133 enast(new_wm->sagv.wm0.enable),
6134 enast(new_wm->sagv.trans_wm.enable));
6135
6136 drm_dbg_kms(&dev_priv->drm,
6137 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6138 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6139 plane->base.base.id, plane->base.name,
6140 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6141 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6142 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6143 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6144 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6145 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6146 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6147 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6148 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6149 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6150 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6151 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6152 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6153 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6154 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6155 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6156 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6157 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6158 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6159 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6160 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6161 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6162
6163 drm_dbg_kms(&dev_priv->drm,
6164 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6165 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6166 plane->base.base.id, plane->base.name,
6167 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6168 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6169 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6170 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6171 old_wm->trans_wm.blocks,
6172 old_wm->sagv.wm0.blocks,
6173 old_wm->sagv.trans_wm.blocks,
6174 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6175 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6176 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6177 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6178 new_wm->trans_wm.blocks,
6179 new_wm->sagv.wm0.blocks,
6180 new_wm->sagv.trans_wm.blocks);
6181
6182 drm_dbg_kms(&dev_priv->drm,
6183 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6184 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6185 plane->base.base.id, plane->base.name,
6186 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6187 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6188 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6189 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6190 old_wm->trans_wm.min_ddb_alloc,
6191 old_wm->sagv.wm0.min_ddb_alloc,
6192 old_wm->sagv.trans_wm.min_ddb_alloc,
6193 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6194 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6195 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6196 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6197 new_wm->trans_wm.min_ddb_alloc,
6198 new_wm->sagv.wm0.min_ddb_alloc,
6199 new_wm->sagv.trans_wm.min_ddb_alloc);
6200 }
6201 }
6202}
6203
6204static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6205 const struct skl_pipe_wm *old_pipe_wm,
6206 const struct skl_pipe_wm *new_pipe_wm)
6207{
6208 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6209 int level, max_level = ilk_wm_max_level(i915);
6210
6211 for (level = 0; level <= max_level; level++) {
6212 /*
6213 * We don't check uv_wm as the hardware doesn't actually
6214 * use it. It only gets used for calculating the required
6215 * ddb allocation.
6216 */
6217 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6218 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6219 return false;
6220 }
6221
6222 if (HAS_HW_SAGV_WM(i915)) {
6223 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6224 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6225
6226 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6227 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6228 return false;
6229 }
6230
6231 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6232 skl_plane_trans_wm(new_pipe_wm, plane->id));
6233}
6234
6235/*
6236 * To make sure the cursor watermark registers are always consistent
6237 * with our computed state the following scenario needs special
6238 * treatment:
6239 *
6240 * 1. enable cursor
6241 * 2. move cursor entirely offscreen
6242 * 3. disable cursor
6243 *
6244 * Step 2. does call .disable_plane() but does not zero the watermarks
6245 * (since we consider an offscreen cursor still active for the purposes
6246 * of watermarks). Step 3. would not normally call .disable_plane()
6247 * because the actual plane visibility isn't changing, and we don't
6248 * deallocate the cursor ddb until the pipe gets disabled. So we must
6249 * force step 3. to call .disable_plane() to update the watermark
6250 * registers properly.
6251 *
6252 * Other planes do not suffer from this issues as their watermarks are
6253 * calculated based on the actual plane visibility. The only time this
6254 * can trigger for the other planes is during the initial readout as the
6255 * default value of the watermarks registers is not zero.
6256 */
6257static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6258 struct intel_crtc *crtc)
6259{
6260 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6261 const struct intel_crtc_state *old_crtc_state =
6262 intel_atomic_get_old_crtc_state(state, crtc);
6263 struct intel_crtc_state *new_crtc_state =
6264 intel_atomic_get_new_crtc_state(state, crtc);
6265 struct intel_plane *plane;
6266
6267 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6268 struct intel_plane_state *plane_state;
6269 enum plane_id plane_id = plane->id;
6270
6271 /*
6272 * Force a full wm update for every plane on modeset.
6273 * Required because the reset value of the wm registers
6274 * is non-zero, whereas we want all disabled planes to
6275 * have zero watermarks. So if we turn off the relevant
6276 * power well the hardware state will go out of sync
6277 * with the software state.
6278 */
6279 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6280 skl_plane_selected_wm_equals(plane,
6281 &old_crtc_state->wm.skl.optimal,
6282 &new_crtc_state->wm.skl.optimal))
6283 continue;
6284
6285 plane_state = intel_atomic_get_plane_state(state, plane);
6286 if (IS_ERR(plane_state))
6287 return PTR_ERR(plane_state);
6288
6289 new_crtc_state->update_planes |= BIT(plane_id);
6290 }
6291
6292 return 0;
6293}
6294
6295static int
6296skl_compute_wm(struct intel_atomic_state *state)
6297{
6298 struct intel_crtc *crtc;
6299 struct intel_crtc_state *new_crtc_state;
6300 int ret, i;
6301
6302 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6303 ret = skl_build_pipe_wm(state, crtc);
6304 if (ret)
6305 return ret;
6306 }
6307
6308 ret = skl_compute_ddb(state);
6309 if (ret)
6310 return ret;
6311
6312 ret = intel_compute_sagv_mask(state);
6313 if (ret)
6314 return ret;
6315
6316 /*
6317 * skl_compute_ddb() will have adjusted the final watermarks
6318 * based on how much ddb is available. Now we can actually
6319 * check if the final watermarks changed.
6320 */
6321 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6322 ret = skl_wm_add_affected_planes(state, crtc);
6323 if (ret)
6324 return ret;
6325 }
6326
6327 skl_print_wm_changes(state);
6328
6329 return 0;
6330}
6331
6332static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6333 struct intel_wm_config *config)
6334{
6335 struct intel_crtc *crtc;
6336
6337 /* Compute the currently _active_ config */
6338 for_each_intel_crtc(&dev_priv->drm, crtc) {
6339 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6340
6341 if (!wm->pipe_enabled)
6342 continue;
6343
6344 config->sprites_enabled |= wm->sprites_enabled;
6345 config->sprites_scaled |= wm->sprites_scaled;
6346 config->num_pipes_active++;
6347 }
6348}
6349
6350static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6351{
6352 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6353 struct ilk_wm_maximums max;
6354 struct intel_wm_config config = {};
6355 struct ilk_wm_values results = {};
6356 enum intel_ddb_partitioning partitioning;
6357
6358 ilk_compute_wm_config(dev_priv, &config);
6359
6360 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6361 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6362
6363 /* 5/6 split only in single pipe config on IVB+ */
6364 if (DISPLAY_VER(dev_priv) >= 7 &&
6365 config.num_pipes_active == 1 && config.sprites_enabled) {
6366 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6367 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6368
6369 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6370 } else {
6371 best_lp_wm = &lp_wm_1_2;
6372 }
6373
6374 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6375 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6376
6377 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6378
6379 ilk_write_wm_values(dev_priv, &results);
6380}
6381
6382static void ilk_initial_watermarks(struct intel_atomic_state *state,
6383 struct intel_crtc *crtc)
6384{
6385 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6386 const struct intel_crtc_state *crtc_state =
6387 intel_atomic_get_new_crtc_state(state, crtc);
6388
6389 mutex_lock(&dev_priv->wm.wm_mutex);
6390 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6391 ilk_program_watermarks(dev_priv);
6392 mutex_unlock(&dev_priv->wm.wm_mutex);
6393}
6394
6395static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6396 struct intel_crtc *crtc)
6397{
6398 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6399 const struct intel_crtc_state *crtc_state =
6400 intel_atomic_get_new_crtc_state(state, crtc);
6401
6402 if (!crtc_state->wm.need_postvbl_update)
6403 return;
6404
6405 mutex_lock(&dev_priv->wm.wm_mutex);
6406 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6407 ilk_program_watermarks(dev_priv);
6408 mutex_unlock(&dev_priv->wm.wm_mutex);
6409}
6410
6411static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6412{
6413 level->enable = val & PLANE_WM_EN;
6414 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6415 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6416 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6417}
6418
6419void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6420 struct skl_pipe_wm *out)
6421{
6422 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6423 enum pipe pipe = crtc->pipe;
6424 int level, max_level;
6425 enum plane_id plane_id;
6426 u32 val;
6427
6428 max_level = ilk_wm_max_level(dev_priv);
6429
6430 for_each_plane_id_on_crtc(crtc, plane_id) {
6431 struct skl_plane_wm *wm = &out->planes[plane_id];
6432
6433 for (level = 0; level <= max_level; level++) {
6434 if (plane_id != PLANE_CURSOR)
6435 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6436 else
6437 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6438
6439 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6440 }
6441
6442 if (plane_id != PLANE_CURSOR)
6443 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6444 else
6445 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6446
6447 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6448
6449 if (HAS_HW_SAGV_WM(dev_priv)) {
6450 if (plane_id != PLANE_CURSOR)
6451 val = intel_uncore_read(&dev_priv->uncore,
6452 PLANE_WM_SAGV(pipe, plane_id));
6453 else
6454 val = intel_uncore_read(&dev_priv->uncore,
6455 CUR_WM_SAGV(pipe));
6456
6457 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6458
6459 if (plane_id != PLANE_CURSOR)
6460 val = intel_uncore_read(&dev_priv->uncore,
6461 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6462 else
6463 val = intel_uncore_read(&dev_priv->uncore,
6464 CUR_WM_SAGV_TRANS(pipe));
6465
6466 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6467 } else if (DISPLAY_VER(dev_priv) >= 12) {
6468 wm->sagv.wm0 = wm->wm[0];
6469 wm->sagv.trans_wm = wm->trans_wm;
6470 }
6471 }
6472}
6473
6474void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6475{
6476 struct intel_dbuf_state *dbuf_state =
6477 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6478 struct intel_crtc *crtc;
6479
6480 if (IS_ALDERLAKE_P(dev_priv))
6481 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6482
6483 for_each_intel_crtc(&dev_priv->drm, crtc) {
6484 struct intel_crtc_state *crtc_state =
6485 to_intel_crtc_state(crtc->base.state);
6486 enum pipe pipe = crtc->pipe;
6487 unsigned int mbus_offset;
6488 enum plane_id plane_id;
6489
6490 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6491 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6492
6493 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6494
6495 for_each_plane_id_on_crtc(crtc, plane_id) {
6496 struct skl_ddb_entry *ddb_y =
6497 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6498 struct skl_ddb_entry *ddb_uv =
6499 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6500
6501 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6502 plane_id, ddb_y, ddb_uv);
6503
6504 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6505 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6506 }
6507
6508 dbuf_state->slices[pipe] =
6509 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6510
6511 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6512
6513 /*
6514 * Used for checking overlaps, so we need absolute
6515 * offsets instead of MBUS relative offsets.
6516 */
6517 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_state->slices[pipe]);
6518 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6519 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6520
6521 drm_dbg_kms(&dev_priv->drm,
6522 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6523 crtc->base.base.id, crtc->base.name,
6524 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6525 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6526 yesno(dbuf_state->joined_mbus));
6527 }
6528
6529 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6530}
6531
6532static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6533{
6534 struct drm_device *dev = crtc->base.dev;
6535 struct drm_i915_private *dev_priv = to_i915(dev);
6536 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6537 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6538 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6539 enum pipe pipe = crtc->pipe;
6540
6541 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6542
6543 memset(active, 0, sizeof(*active));
6544
6545 active->pipe_enabled = crtc->active;
6546
6547 if (active->pipe_enabled) {
6548 u32 tmp = hw->wm_pipe[pipe];
6549
6550 /*
6551 * For active pipes LP0 watermark is marked as
6552 * enabled, and LP1+ watermaks as disabled since
6553 * we can't really reverse compute them in case
6554 * multiple pipes are active.
6555 */
6556 active->wm[0].enable = true;
6557 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6558 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6559 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
6560 } else {
6561 int level, max_level = ilk_wm_max_level(dev_priv);
6562
6563 /*
6564 * For inactive pipes, all watermark levels
6565 * should be marked as enabled but zeroed,
6566 * which is what we'd compute them to.
6567 */
6568 for (level = 0; level <= max_level; level++)
6569 active->wm[level].enable = true;
6570 }
6571
6572 crtc->wm.active.ilk = *active;
6573}
6574
6575#define _FW_WM(value, plane) \
6576 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6577#define _FW_WM_VLV(value, plane) \
6578 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6579
6580static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6581 struct g4x_wm_values *wm)
6582{
6583 u32 tmp;
6584
6585 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6586 wm->sr.plane = _FW_WM(tmp, SR);
6587 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6588 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6589 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6590
6591 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6592 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6593 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6594 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6595 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6596 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6597 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6598
6599 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6600 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6601 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6602 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6603 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6604}
6605
6606static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6607 struct vlv_wm_values *wm)
6608{
6609 enum pipe pipe;
6610 u32 tmp;
6611
6612 for_each_pipe(dev_priv, pipe) {
6613 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6614
6615 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6616 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6617 wm->ddl[pipe].plane[PLANE_CURSOR] =
6618 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6619 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6620 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6621 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6622 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6623 }
6624
6625 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6626 wm->sr.plane = _FW_WM(tmp, SR);
6627 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6628 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6629 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6630
6631 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6632 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6633 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6634 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6635
6636 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6637 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6638
6639 if (IS_CHERRYVIEW(dev_priv)) {
6640 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6641 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6642 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6643
6644 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6645 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6646 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6647
6648 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6649 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6650 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6651
6652 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6653 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6654 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6655 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6656 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6657 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6658 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6659 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6660 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6661 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6662 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6663 } else {
6664 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6665 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6666 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6667
6668 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6669 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6670 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6671 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6672 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6673 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6674 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6675 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6676 }
6677}
6678
6679#undef _FW_WM
6680#undef _FW_WM_VLV
6681
6682void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6683{
6684 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6685 struct intel_crtc *crtc;
6686
6687 g4x_read_wm_values(dev_priv, wm);
6688
6689 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6690
6691 for_each_intel_crtc(&dev_priv->drm, crtc) {
6692 struct intel_crtc_state *crtc_state =
6693 to_intel_crtc_state(crtc->base.state);
6694 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6695 struct g4x_pipe_wm *raw;
6696 enum pipe pipe = crtc->pipe;
6697 enum plane_id plane_id;
6698 int level, max_level;
6699
6700 active->cxsr = wm->cxsr;
6701 active->hpll_en = wm->hpll_en;
6702 active->fbc_en = wm->fbc_en;
6703
6704 active->sr = wm->sr;
6705 active->hpll = wm->hpll;
6706
6707 for_each_plane_id_on_crtc(crtc, plane_id) {
6708 active->wm.plane[plane_id] =
6709 wm->pipe[pipe].plane[plane_id];
6710 }
6711
6712 if (wm->cxsr && wm->hpll_en)
6713 max_level = G4X_WM_LEVEL_HPLL;
6714 else if (wm->cxsr)
6715 max_level = G4X_WM_LEVEL_SR;
6716 else
6717 max_level = G4X_WM_LEVEL_NORMAL;
6718
6719 level = G4X_WM_LEVEL_NORMAL;
6720 raw = &crtc_state->wm.g4x.raw[level];
6721 for_each_plane_id_on_crtc(crtc, plane_id)
6722 raw->plane[plane_id] = active->wm.plane[plane_id];
6723
6724 if (++level > max_level)
6725 goto out;
6726
6727 raw = &crtc_state->wm.g4x.raw[level];
6728 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6729 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6730 raw->plane[PLANE_SPRITE0] = 0;
6731 raw->fbc = active->sr.fbc;
6732
6733 if (++level > max_level)
6734 goto out;
6735
6736 raw = &crtc_state->wm.g4x.raw[level];
6737 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6738 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6739 raw->plane[PLANE_SPRITE0] = 0;
6740 raw->fbc = active->hpll.fbc;
6741
6742 out:
6743 for_each_plane_id_on_crtc(crtc, plane_id)
6744 g4x_raw_plane_wm_set(crtc_state, level,
6745 plane_id, USHRT_MAX);
6746 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6747
6748 crtc_state->wm.g4x.optimal = *active;
6749 crtc_state->wm.g4x.intermediate = *active;
6750
6751 drm_dbg_kms(&dev_priv->drm,
6752 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6753 pipe_name(pipe),
6754 wm->pipe[pipe].plane[PLANE_PRIMARY],
6755 wm->pipe[pipe].plane[PLANE_CURSOR],
6756 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6757 }
6758
6759 drm_dbg_kms(&dev_priv->drm,
6760 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6761 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6762 drm_dbg_kms(&dev_priv->drm,
6763 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6764 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6765 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6766 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6767}
6768
6769void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6770{
6771 struct intel_plane *plane;
6772 struct intel_crtc *crtc;
6773
6774 mutex_lock(&dev_priv->wm.wm_mutex);
6775
6776 for_each_intel_plane(&dev_priv->drm, plane) {
6777 struct intel_crtc *crtc =
6778 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6779 struct intel_crtc_state *crtc_state =
6780 to_intel_crtc_state(crtc->base.state);
6781 struct intel_plane_state *plane_state =
6782 to_intel_plane_state(plane->base.state);
6783 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6784 enum plane_id plane_id = plane->id;
6785 int level;
6786
6787 if (plane_state->uapi.visible)
6788 continue;
6789
6790 for (level = 0; level < 3; level++) {
6791 struct g4x_pipe_wm *raw =
6792 &crtc_state->wm.g4x.raw[level];
6793
6794 raw->plane[plane_id] = 0;
6795 wm_state->wm.plane[plane_id] = 0;
6796 }
6797
6798 if (plane_id == PLANE_PRIMARY) {
6799 for (level = 0; level < 3; level++) {
6800 struct g4x_pipe_wm *raw =
6801 &crtc_state->wm.g4x.raw[level];
6802 raw->fbc = 0;
6803 }
6804
6805 wm_state->sr.fbc = 0;
6806 wm_state->hpll.fbc = 0;
6807 wm_state->fbc_en = false;
6808 }
6809 }
6810
6811 for_each_intel_crtc(&dev_priv->drm, crtc) {
6812 struct intel_crtc_state *crtc_state =
6813 to_intel_crtc_state(crtc->base.state);
6814
6815 crtc_state->wm.g4x.intermediate =
6816 crtc_state->wm.g4x.optimal;
6817 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6818 }
6819
6820 g4x_program_watermarks(dev_priv);
6821
6822 mutex_unlock(&dev_priv->wm.wm_mutex);
6823}
6824
6825void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6826{
6827 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6828 struct intel_crtc *crtc;
6829 u32 val;
6830
6831 vlv_read_wm_values(dev_priv, wm);
6832
6833 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6834 wm->level = VLV_WM_LEVEL_PM2;
6835
6836 if (IS_CHERRYVIEW(dev_priv)) {
6837 vlv_punit_get(dev_priv);
6838
6839 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6840 if (val & DSP_MAXFIFO_PM5_ENABLE)
6841 wm->level = VLV_WM_LEVEL_PM5;
6842
6843 /*
6844 * If DDR DVFS is disabled in the BIOS, Punit
6845 * will never ack the request. So if that happens
6846 * assume we don't have to enable/disable DDR DVFS
6847 * dynamically. To test that just set the REQ_ACK
6848 * bit to poke the Punit, but don't change the
6849 * HIGH/LOW bits so that we don't actually change
6850 * the current state.
6851 */
6852 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6853 val |= FORCE_DDR_FREQ_REQ_ACK;
6854 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6855
6856 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6857 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6858 drm_dbg_kms(&dev_priv->drm,
6859 "Punit not acking DDR DVFS request, "
6860 "assuming DDR DVFS is disabled\n");
6861 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6862 } else {
6863 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6864 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6865 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6866 }
6867
6868 vlv_punit_put(dev_priv);
6869 }
6870
6871 for_each_intel_crtc(&dev_priv->drm, crtc) {
6872 struct intel_crtc_state *crtc_state =
6873 to_intel_crtc_state(crtc->base.state);
6874 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6875 const struct vlv_fifo_state *fifo_state =
6876 &crtc_state->wm.vlv.fifo_state;
6877 enum pipe pipe = crtc->pipe;
6878 enum plane_id plane_id;
6879 int level;
6880
6881 vlv_get_fifo_size(crtc_state);
6882
6883 active->num_levels = wm->level + 1;
6884 active->cxsr = wm->cxsr;
6885
6886 for (level = 0; level < active->num_levels; level++) {
6887 struct g4x_pipe_wm *raw =
6888 &crtc_state->wm.vlv.raw[level];
6889
6890 active->sr[level].plane = wm->sr.plane;
6891 active->sr[level].cursor = wm->sr.cursor;
6892
6893 for_each_plane_id_on_crtc(crtc, plane_id) {
6894 active->wm[level].plane[plane_id] =
6895 wm->pipe[pipe].plane[plane_id];
6896
6897 raw->plane[plane_id] =
6898 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6899 fifo_state->plane[plane_id]);
6900 }
6901 }
6902
6903 for_each_plane_id_on_crtc(crtc, plane_id)
6904 vlv_raw_plane_wm_set(crtc_state, level,
6905 plane_id, USHRT_MAX);
6906 vlv_invalidate_wms(crtc, active, level);
6907
6908 crtc_state->wm.vlv.optimal = *active;
6909 crtc_state->wm.vlv.intermediate = *active;
6910
6911 drm_dbg_kms(&dev_priv->drm,
6912 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6913 pipe_name(pipe),
6914 wm->pipe[pipe].plane[PLANE_PRIMARY],
6915 wm->pipe[pipe].plane[PLANE_CURSOR],
6916 wm->pipe[pipe].plane[PLANE_SPRITE0],
6917 wm->pipe[pipe].plane[PLANE_SPRITE1]);
6918 }
6919
6920 drm_dbg_kms(&dev_priv->drm,
6921 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6922 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6923}
6924
6925void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6926{
6927 struct intel_plane *plane;
6928 struct intel_crtc *crtc;
6929
6930 mutex_lock(&dev_priv->wm.wm_mutex);
6931
6932 for_each_intel_plane(&dev_priv->drm, plane) {
6933 struct intel_crtc *crtc =
6934 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6935 struct intel_crtc_state *crtc_state =
6936 to_intel_crtc_state(crtc->base.state);
6937 struct intel_plane_state *plane_state =
6938 to_intel_plane_state(plane->base.state);
6939 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6940 const struct vlv_fifo_state *fifo_state =
6941 &crtc_state->wm.vlv.fifo_state;
6942 enum plane_id plane_id = plane->id;
6943 int level;
6944
6945 if (plane_state->uapi.visible)
6946 continue;
6947
6948 for (level = 0; level < wm_state->num_levels; level++) {
6949 struct g4x_pipe_wm *raw =
6950 &crtc_state->wm.vlv.raw[level];
6951
6952 raw->plane[plane_id] = 0;
6953
6954 wm_state->wm[level].plane[plane_id] =
6955 vlv_invert_wm_value(raw->plane[plane_id],
6956 fifo_state->plane[plane_id]);
6957 }
6958 }
6959
6960 for_each_intel_crtc(&dev_priv->drm, crtc) {
6961 struct intel_crtc_state *crtc_state =
6962 to_intel_crtc_state(crtc->base.state);
6963
6964 crtc_state->wm.vlv.intermediate =
6965 crtc_state->wm.vlv.optimal;
6966 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6967 }
6968
6969 vlv_program_watermarks(dev_priv);
6970
6971 mutex_unlock(&dev_priv->wm.wm_mutex);
6972}
6973
6974/*
6975 * FIXME should probably kill this and improve
6976 * the real watermark readout/sanitation instead
6977 */
6978static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6979{
6980 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
6981 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
6982 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
6983
6984 /*
6985 * Don't touch WM1S_LP_EN here.
6986 * Doing so could cause underruns.
6987 */
6988}
6989
6990void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6991{
6992 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6993 struct intel_crtc *crtc;
6994
6995 ilk_init_lp_watermarks(dev_priv);
6996
6997 for_each_intel_crtc(&dev_priv->drm, crtc)
6998 ilk_pipe_wm_get_hw_state(crtc);
6999
7000 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7001 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7002 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7003
7004 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7005 if (DISPLAY_VER(dev_priv) >= 7) {
7006 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7007 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7008 }
7009
7010 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7011 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7012 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7013 else if (IS_IVYBRIDGE(dev_priv))
7014 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7015 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7016
7017 hw->enable_fbc_wm =
7018 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7019}
7020
7021/**
7022 * intel_update_watermarks - update FIFO watermark values based on current modes
7023 * @crtc: the #intel_crtc on which to compute the WM
7024 *
7025 * Calculate watermark values for the various WM regs based on current mode
7026 * and plane configuration.
7027 *
7028 * There are several cases to deal with here:
7029 * - normal (i.e. non-self-refresh)
7030 * - self-refresh (SR) mode
7031 * - lines are large relative to FIFO size (buffer can hold up to 2)
7032 * - lines are small relative to FIFO size (buffer can hold more than 2
7033 * lines), so need to account for TLB latency
7034 *
7035 * The normal calculation is:
7036 * watermark = dotclock * bytes per pixel * latency
7037 * where latency is platform & configuration dependent (we assume pessimal
7038 * values here).
7039 *
7040 * The SR calculation is:
7041 * watermark = (trunc(latency/line time)+1) * surface width *
7042 * bytes per pixel
7043 * where
7044 * line time = htotal / dotclock
7045 * surface width = hdisplay for normal plane and 64 for cursor
7046 * and latency is assumed to be high, as above.
7047 *
7048 * The final value programmed to the register should always be rounded up,
7049 * and include an extra 2 entries to account for clock crossings.
7050 *
7051 * We don't use the sprite, so we can ignore that. And on Crestline we have
7052 * to set the non-SR watermarks to 8.
7053 */
7054void intel_update_watermarks(struct intel_crtc *crtc)
7055{
7056 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7057
7058 if (dev_priv->display.update_wm)
7059 dev_priv->display.update_wm(crtc);
7060}
7061
7062void intel_enable_ipc(struct drm_i915_private *dev_priv)
7063{
7064 u32 val;
7065
7066 if (!HAS_IPC(dev_priv))
7067 return;
7068
7069 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7070
7071 if (dev_priv->ipc_enabled)
7072 val |= DISP_IPC_ENABLE;
7073 else
7074 val &= ~DISP_IPC_ENABLE;
7075
7076 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7077}
7078
7079static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7080{
7081 /* Display WA #0477 WaDisableIPC: skl */
7082 if (IS_SKYLAKE(dev_priv))
7083 return false;
7084
7085 /* Display WA #1141: SKL:all KBL:all CFL */
7086 if (IS_KABYLAKE(dev_priv) ||
7087 IS_COFFEELAKE(dev_priv) ||
7088 IS_COMETLAKE(dev_priv))
7089 return dev_priv->dram_info.symmetric_memory;
7090
7091 return true;
7092}
7093
7094void intel_init_ipc(struct drm_i915_private *dev_priv)
7095{
7096 if (!HAS_IPC(dev_priv))
7097 return;
7098
7099 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7100
7101 intel_enable_ipc(dev_priv);
7102}
7103
7104static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7105{
7106 /*
7107 * On Ibex Peak and Cougar Point, we need to disable clock
7108 * gating for the panel power sequencer or it will fail to
7109 * start up when no ports are active.
7110 */
7111 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7112}
7113
7114static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7115{
7116 enum pipe pipe;
7117
7118 for_each_pipe(dev_priv, pipe) {
7119 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7120 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7121 DISPPLANE_TRICKLE_FEED_DISABLE);
7122
7123 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7124 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7125 }
7126}
7127
7128static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7129{
7130 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7131
7132 /*
7133 * Required for FBC
7134 * WaFbcDisableDpfcClockGating:ilk
7135 */
7136 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7137 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7138 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7139
7140 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7141 MARIUNIT_CLOCK_GATE_DISABLE |
7142 SVSMUNIT_CLOCK_GATE_DISABLE);
7143 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7144 VFMUNIT_CLOCK_GATE_DISABLE);
7145
7146 /*
7147 * According to the spec the following bits should be set in
7148 * order to enable memory self-refresh
7149 * The bit 22/21 of 0x42004
7150 * The bit 5 of 0x42020
7151 * The bit 15 of 0x45000
7152 */
7153 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7154 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7155 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7156 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7157 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7158 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7159 DISP_FBC_WM_DIS));
7160
7161 /*
7162 * Based on the document from hardware guys the following bits
7163 * should be set unconditionally in order to enable FBC.
7164 * The bit 22 of 0x42000
7165 * The bit 22 of 0x42004
7166 * The bit 7,8,9 of 0x42020.
7167 */
7168 if (IS_IRONLAKE_M(dev_priv)) {
7169 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7170 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7171 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7172 ILK_FBCQ_DIS);
7173 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7174 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7175 ILK_DPARB_GATE);
7176 }
7177
7178 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7179
7180 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7181 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7182 ILK_ELPIN_409_SELECT);
7183
7184 g4x_disable_trickle_feed(dev_priv);
7185
7186 ibx_init_clock_gating(dev_priv);
7187}
7188
7189static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7190{
7191 enum pipe pipe;
7192 u32 val;
7193
7194 /*
7195 * On Ibex Peak and Cougar Point, we need to disable clock
7196 * gating for the panel power sequencer or it will fail to
7197 * start up when no ports are active.
7198 */
7199 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7200 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7201 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7202 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7203 DPLS_EDP_PPS_FIX_DIS);
7204 /* The below fixes the weird display corruption, a few pixels shifted
7205 * downward, on (only) LVDS of some HP laptops with IVY.
7206 */
7207 for_each_pipe(dev_priv, pipe) {
7208 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7209 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7210 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7211 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7212 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7213 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7214 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7215 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7216 }
7217 /* WADP0ClockGatingDisable */
7218 for_each_pipe(dev_priv, pipe) {
7219 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7220 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7221 }
7222}
7223
7224static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7225{
7226 u32 tmp;
7227
7228 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7229 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
7230 drm_dbg_kms(&dev_priv->drm,
7231 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7232 tmp);
7233}
7234
7235static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7236{
7237 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7238
7239 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7240
7241 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7242 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7243 ILK_ELPIN_409_SELECT);
7244
7245 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7246 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7247 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7248 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7249
7250 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7251 * gating disable must be set. Failure to set it results in
7252 * flickering pixels due to Z write ordering failures after
7253 * some amount of runtime in the Mesa "fire" demo, and Unigine
7254 * Sanctuary and Tropics, and apparently anything else with
7255 * alpha test or pixel discard.
7256 *
7257 * According to the spec, bit 11 (RCCUNIT) must also be set,
7258 * but we didn't debug actual testcases to find it out.
7259 *
7260 * WaDisableRCCUnitClockGating:snb
7261 * WaDisableRCPBUnitClockGating:snb
7262 */
7263 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7264 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7265 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7266
7267 /*
7268 * According to the spec the following bits should be
7269 * set in order to enable memory self-refresh and fbc:
7270 * The bit21 and bit22 of 0x42000
7271 * The bit21 and bit22 of 0x42004
7272 * The bit5 and bit7 of 0x42020
7273 * The bit14 of 0x70180
7274 * The bit14 of 0x71180
7275 *
7276 * WaFbcAsynchFlipDisableFbcQueue:snb
7277 */
7278 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7279 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7280 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7281 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7282 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7283 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7284 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7285 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7286 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7287 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7288
7289 g4x_disable_trickle_feed(dev_priv);
7290
7291 cpt_init_clock_gating(dev_priv);
7292
7293 gen6_check_mch_setup(dev_priv);
7294}
7295
7296static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7297{
7298 /*
7299 * TODO: this bit should only be enabled when really needed, then
7300 * disabled when not needed anymore in order to save power.
7301 */
7302 if (HAS_PCH_LPT_LP(dev_priv))
7303 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7304 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7305 PCH_LP_PARTITION_LEVEL_DISABLE);
7306
7307 /* WADPOClockGatingDisable:hsw */
7308 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7309 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7310 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7311}
7312
7313static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7314{
7315 if (HAS_PCH_LPT_LP(dev_priv)) {
7316 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7317
7318 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7319 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7320 }
7321}
7322
7323static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7324 int general_prio_credits,
7325 int high_prio_credits)
7326{
7327 u32 misccpctl;
7328 u32 val;
7329
7330 /* WaTempDisableDOPClkGating:bdw */
7331 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7332 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7333
7334 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7335 val &= ~L3_PRIO_CREDITS_MASK;
7336 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7337 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7338 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7339
7340 /*
7341 * Wait at least 100 clocks before re-enabling clock gating.
7342 * See the definition of L3SQCREG1 in BSpec.
7343 */
7344 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7345 udelay(1);
7346 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7347}
7348
7349static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7350{
7351 /* Wa_1409120013:icl,ehl */
7352 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7353 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7354
7355 /* This is not an Wa. Enable to reduce Sampler power */
7356 intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7357 intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
7358
7359 /*Wa_14010594013:icl, ehl */
7360 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7361 0, CNL_DELAY_PMRSP);
7362}
7363
7364static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7365{
7366 /* Wa_1409120013:tgl,rkl,adl_s,dg1 */
7367 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
7368 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7369
7370 /* Wa_1409825376:tgl (pre-prod)*/
7371 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B1))
7372 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7373 TGL_VRH_GATING_DIS);
7374
7375 /* Wa_14011059788:tgl,rkl,adl_s,dg1 */
7376 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7377 0, DFR_DISABLE);
7378
7379 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7380 if (DISPLAY_VER(dev_priv) == 12)
7381 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7382 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7383}
7384
7385static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7386{
7387 gen12lp_init_clock_gating(dev_priv);
7388
7389 /* Wa_22011091694:adlp */
7390 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7391}
7392
7393static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7394{
7395 gen12lp_init_clock_gating(dev_priv);
7396
7397 /* Wa_1409836686:dg1[a0] */
7398 if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0))
7399 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7400 DPT_GATING_DIS);
7401}
7402
7403static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7404{
7405 if (!HAS_PCH_CNP(dev_priv))
7406 return;
7407
7408 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7409 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7410 CNP_PWM_CGE_GATING_DISABLE);
7411}
7412
7413static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
7414{
7415 u32 val;
7416 cnp_init_clock_gating(dev_priv);
7417
7418 /* This is not an Wa. Enable for better image quality */
7419 intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3,
7420 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7421
7422 /* WaEnableChickenDCPR:cnl */
7423 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7424 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
7425
7426 /*
7427 * WaFbcWakeMemOn:cnl
7428 * Display WA #0859: cnl
7429 */
7430 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7431 DISP_FBC_MEMORY_WAKE);
7432
7433 val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE);
7434 /* ReadHitWriteOnlyDisable:cnl */
7435 val |= RCCUNIT_CLKGATE_DIS;
7436 intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val);
7437
7438 /* Wa_2201832410:cnl */
7439 val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE);
7440 val |= GWUNIT_CLKGATE_DIS;
7441 intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val);
7442
7443 /* WaDisableVFclkgate:cnl */
7444 /* WaVFUnitClockGatingDisable:cnl */
7445 val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE);
7446 val |= VFUNIT_CLKGATE_DIS;
7447 intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val);
7448}
7449
7450static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7451{
7452 cnp_init_clock_gating(dev_priv);
7453 gen9_init_clock_gating(dev_priv);
7454
7455 /* WAC6entrylatency:cfl */
7456 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7457 FBC_LLC_FULLY_OPEN);
7458
7459 /*
7460 * WaFbcTurnOffFbcWatermark:cfl
7461 * Display WA #0562: cfl
7462 */
7463 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7464 DISP_FBC_WM_DIS);
7465
7466 /*
7467 * WaFbcNukeOnHostModify:cfl
7468 * Display WA #0873: cfl
7469 */
7470 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7471 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7472}
7473
7474static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7475{
7476 gen9_init_clock_gating(dev_priv);
7477
7478 /* WAC6entrylatency:kbl */
7479 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7480 FBC_LLC_FULLY_OPEN);
7481
7482 /* WaDisableSDEUnitClockGating:kbl */
7483 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7484 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7485 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7486
7487 /* WaDisableGamClockGating:kbl */
7488 if (IS_KBL_GT_STEP(dev_priv, 0, STEP_B0))
7489 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7490 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7491
7492 /*
7493 * WaFbcTurnOffFbcWatermark:kbl
7494 * Display WA #0562: kbl
7495 */
7496 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7497 DISP_FBC_WM_DIS);
7498
7499 /*
7500 * WaFbcNukeOnHostModify:kbl
7501 * Display WA #0873: kbl
7502 */
7503 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7504 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7505}
7506
7507static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7508{
7509 gen9_init_clock_gating(dev_priv);
7510
7511 /* WaDisableDopClockGating:skl */
7512 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7513 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7514
7515 /* WAC6entrylatency:skl */
7516 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7517 FBC_LLC_FULLY_OPEN);
7518
7519 /*
7520 * WaFbcTurnOffFbcWatermark:skl
7521 * Display WA #0562: skl
7522 */
7523 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7524 DISP_FBC_WM_DIS);
7525
7526 /*
7527 * WaFbcNukeOnHostModify:skl
7528 * Display WA #0873: skl
7529 */
7530 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7531 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7532
7533 /*
7534 * WaFbcHighMemBwCorruptionAvoidance:skl
7535 * Display WA #0883: skl
7536 */
7537 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
7538 ILK_DPFC_DISABLE_DUMMY0);
7539}
7540
7541static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7542{
7543 enum pipe pipe;
7544
7545 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7546 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7547 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7548 HSW_FBCQ_DIS);
7549
7550 /* WaSwitchSolVfFArbitrationPriority:bdw */
7551 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7552
7553 /* WaPsrDPAMaskVBlankInSRD:bdw */
7554 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7555 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7556
7557 for_each_pipe(dev_priv, pipe) {
7558 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7559 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7560 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7561 BDW_DPRS_MASK_VBLANK_SRD);
7562
7563 /* Undocumented but fixes async flip + VT-d corruption */
7564 if (intel_vtd_active())
7565 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7566 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7567 }
7568
7569 /* WaVSRefCountFullforceMissDisable:bdw */
7570 /* WaDSRefCountFullforceMissDisable:bdw */
7571 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7572 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7573 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7574
7575 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7576 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7577
7578 /* WaDisableSDEUnitClockGating:bdw */
7579 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7580 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7581
7582 /* WaProgramL3SqcReg1Default:bdw */
7583 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7584
7585 /* WaKVMNotificationOnConfigChange:bdw */
7586 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7587 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7588
7589 lpt_init_clock_gating(dev_priv);
7590
7591 /* WaDisableDopClockGating:bdw
7592 *
7593 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7594 * clock gating.
7595 */
7596 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7597 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7598}
7599
7600static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7601{
7602 enum pipe pipe;
7603
7604 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7605 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7606 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7607 HSW_FBCQ_DIS);
7608
7609 for_each_pipe(dev_priv, pipe) {
7610 /* Undocumented but fixes async flip + VT-d corruption */
7611 if (intel_vtd_active())
7612 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7613 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7614 }
7615
7616 /* This is required by WaCatErrorRejectionIssue:hsw */
7617 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7618 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7619 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7620
7621 /* WaSwitchSolVfFArbitrationPriority:hsw */
7622 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7623
7624 lpt_init_clock_gating(dev_priv);
7625}
7626
7627static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7628{
7629 u32 snpcr;
7630
7631 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7632
7633 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7634 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7635 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7636 ILK_FBCQ_DIS);
7637
7638 /* WaDisableBackToBackFlipFix:ivb */
7639 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7640 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7641 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7642
7643 if (IS_IVB_GT1(dev_priv))
7644 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7645 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7646 else {
7647 /* must write both registers */
7648 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7649 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7650 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7651 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7652 }
7653
7654 /*
7655 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7656 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7657 */
7658 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7659 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7660
7661 /* This is required by WaCatErrorRejectionIssue:ivb */
7662 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7663 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7664 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7665
7666 g4x_disable_trickle_feed(dev_priv);
7667
7668 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7669 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7670 snpcr |= GEN6_MBC_SNPCR_MED;
7671 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7672
7673 if (!HAS_PCH_NOP(dev_priv))
7674 cpt_init_clock_gating(dev_priv);
7675
7676 gen6_check_mch_setup(dev_priv);
7677}
7678
7679static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7680{
7681 /* WaDisableBackToBackFlipFix:vlv */
7682 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7683 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7684 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7685
7686 /* WaDisableDopClockGating:vlv */
7687 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7688 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7689
7690 /* This is required by WaCatErrorRejectionIssue:vlv */
7691 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7692 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7693 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7694
7695 /*
7696 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7697 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7698 */
7699 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7700 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7701
7702 /* WaDisableL3Bank2xClockGate:vlv
7703 * Disabling L3 clock gating- MMIO 940c[25] = 1
7704 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7705 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7706 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7707
7708 /*
7709 * WaDisableVLVClockGating_VBIIssue:vlv
7710 * Disable clock gating on th GCFG unit to prevent a delay
7711 * in the reporting of vblank events.
7712 */
7713 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7714}
7715
7716static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7717{
7718 /* WaVSRefCountFullforceMissDisable:chv */
7719 /* WaDSRefCountFullforceMissDisable:chv */
7720 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7721 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7722 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7723
7724 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7725 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
7726 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7727
7728 /* WaDisableCSUnitClockGating:chv */
7729 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7730 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7731
7732 /* WaDisableSDEUnitClockGating:chv */
7733 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7734 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7735
7736 /*
7737 * WaProgramL3SqcReg1Default:chv
7738 * See gfxspecs/Related Documents/Performance Guide/
7739 * LSQC Setting Recommendations.
7740 */
7741 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7742}
7743
7744static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7745{
7746 u32 dspclk_gate;
7747
7748 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7749 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7750 GS_UNIT_CLOCK_GATE_DISABLE |
7751 CL_UNIT_CLOCK_GATE_DISABLE);
7752 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7753 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7754 OVRUNIT_CLOCK_GATE_DISABLE |
7755 OVCUNIT_CLOCK_GATE_DISABLE;
7756 if (IS_GM45(dev_priv))
7757 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7758 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7759
7760 g4x_disable_trickle_feed(dev_priv);
7761}
7762
7763static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7764{
7765 struct intel_uncore *uncore = &dev_priv->uncore;
7766
7767 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7768 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7769 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7770 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7771 intel_uncore_write16(uncore, DEUC, 0);
7772 intel_uncore_write(uncore,
7773 MI_ARB_STATE,
7774 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7775}
7776
7777static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7778{
7779 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7780 I965_RCC_CLOCK_GATE_DISABLE |
7781 I965_RCPB_CLOCK_GATE_DISABLE |
7782 I965_ISC_CLOCK_GATE_DISABLE |
7783 I965_FBC_CLOCK_GATE_DISABLE);
7784 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7785 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7786 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7787}
7788
7789static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7790{
7791 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7792
7793 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7794 DSTATE_DOT_CLOCK_GATING;
7795 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7796
7797 if (IS_PINEVIEW(dev_priv))
7798 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7799
7800 /* IIR "flip pending" means done if this bit is set */
7801 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7802
7803 /* interrupts should cause a wake up from C3 */
7804 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7805
7806 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7807 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7808
7809 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7810 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7811}
7812
7813static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7814{
7815 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7816
7817 /* interrupts should cause a wake up from C3 */
7818 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7819 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7820
7821 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7822 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7823
7824 /*
7825 * Have FBC ignore 3D activity since we use software
7826 * render tracking, and otherwise a pure 3D workload
7827 * (even if it just renders a single frame and then does
7828 * abosultely nothing) would not allow FBC to recompress
7829 * until a 2D blit occurs.
7830 */
7831 intel_uncore_write(&dev_priv->uncore, SCPD0,
7832 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7833}
7834
7835static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7836{
7837 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7838 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7839 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7840}
7841
7842void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7843{
7844 dev_priv->display.init_clock_gating(dev_priv);
7845}
7846
7847void intel_suspend_hw(struct drm_i915_private *dev_priv)
7848{
7849 if (HAS_PCH_LPT(dev_priv))
7850 lpt_suspend_hw(dev_priv);
7851}
7852
7853static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7854{
7855 drm_dbg_kms(&dev_priv->drm,
7856 "No clock gating settings or workarounds applied.\n");
7857}
7858
7859/**
7860 * intel_init_clock_gating_hooks - setup the clock gating hooks
7861 * @dev_priv: device private
7862 *
7863 * Setup the hooks that configure which clocks of a given platform can be
7864 * gated and also apply various GT and display specific workarounds for these
7865 * platforms. Note that some GT specific workarounds are applied separately
7866 * when GPU contexts or batchbuffers start their execution.
7867 */
7868void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7869{
7870 if (IS_ALDERLAKE_P(dev_priv))
7871 dev_priv->display.init_clock_gating = adlp_init_clock_gating;
7872 else if (IS_DG1(dev_priv))
7873 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7874 else if (GRAPHICS_VER(dev_priv) == 12)
7875 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
7876 else if (GRAPHICS_VER(dev_priv) == 11)
7877 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7878 else if (IS_CANNONLAKE(dev_priv))
7879 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7880 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7881 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7882 else if (IS_SKYLAKE(dev_priv))
7883 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7884 else if (IS_KABYLAKE(dev_priv))
7885 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7886 else if (IS_BROXTON(dev_priv))
7887 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7888 else if (IS_GEMINILAKE(dev_priv))
7889 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7890 else if (IS_BROADWELL(dev_priv))
7891 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7892 else if (IS_CHERRYVIEW(dev_priv))
7893 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7894 else if (IS_HASWELL(dev_priv))
7895 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7896 else if (IS_IVYBRIDGE(dev_priv))
7897 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7898 else if (IS_VALLEYVIEW(dev_priv))
7899 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7900 else if (GRAPHICS_VER(dev_priv) == 6)
7901 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7902 else if (GRAPHICS_VER(dev_priv) == 5)
7903 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7904 else if (IS_G4X(dev_priv))
7905 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7906 else if (IS_I965GM(dev_priv))
7907 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7908 else if (IS_I965G(dev_priv))
7909 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7910 else if (GRAPHICS_VER(dev_priv) == 3)
7911 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7912 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7913 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7914 else if (GRAPHICS_VER(dev_priv) == 2)
7915 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7916 else {
7917 MISSING_CASE(INTEL_DEVID(dev_priv));
7918 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7919 }
7920}
7921
7922/* Set up chip specific power management-related functions */
7923void intel_init_pm(struct drm_i915_private *dev_priv)
7924{
7925 /* For cxsr */
7926 if (IS_PINEVIEW(dev_priv))
7927 pnv_get_mem_freq(dev_priv);
7928 else if (GRAPHICS_VER(dev_priv) == 5)
7929 ilk_get_mem_freq(dev_priv);
7930
7931 if (intel_has_sagv(dev_priv))
7932 skl_setup_sagv_block_time(dev_priv);
7933
7934 /* For FIFO watermark updates */
7935 if (DISPLAY_VER(dev_priv) >= 9) {
7936 skl_setup_wm_latency(dev_priv);
7937 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7938 } else if (HAS_PCH_SPLIT(dev_priv)) {
7939 ilk_setup_wm_latency(dev_priv);
7940
7941 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
7942 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7943 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
7944 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7945 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7946 dev_priv->display.compute_intermediate_wm =
7947 ilk_compute_intermediate_wm;
7948 dev_priv->display.initial_watermarks =
7949 ilk_initial_watermarks;
7950 dev_priv->display.optimize_watermarks =
7951 ilk_optimize_watermarks;
7952 } else {
7953 drm_dbg_kms(&dev_priv->drm,
7954 "Failed to read display plane latency. "
7955 "Disable CxSR\n");
7956 }
7957 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7958 vlv_setup_wm_latency(dev_priv);
7959 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7960 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7961 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7962 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7963 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7964 } else if (IS_G4X(dev_priv)) {
7965 g4x_setup_wm_latency(dev_priv);
7966 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7967 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7968 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7969 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7970 } else if (IS_PINEVIEW(dev_priv)) {
7971 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7972 dev_priv->is_ddr3,
7973 dev_priv->fsb_freq,
7974 dev_priv->mem_freq)) {
7975 drm_info(&dev_priv->drm,
7976 "failed to find known CxSR latency "
7977 "(found ddr%s fsb freq %d, mem freq %d), "
7978 "disabling CxSR\n",
7979 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7980 dev_priv->fsb_freq, dev_priv->mem_freq);
7981 /* Disable CxSR and never update its watermark again */
7982 intel_set_memory_cxsr(dev_priv, false);
7983 dev_priv->display.update_wm = NULL;
7984 } else
7985 dev_priv->display.update_wm = pnv_update_wm;
7986 } else if (DISPLAY_VER(dev_priv) == 4) {
7987 dev_priv->display.update_wm = i965_update_wm;
7988 } else if (DISPLAY_VER(dev_priv) == 3) {
7989 dev_priv->display.update_wm = i9xx_update_wm;
7990 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7991 } else if (DISPLAY_VER(dev_priv) == 2) {
7992 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7993 dev_priv->display.update_wm = i845_update_wm;
7994 dev_priv->display.get_fifo_size = i845_get_fifo_size;
7995 } else {
7996 dev_priv->display.update_wm = i9xx_update_wm;
7997 dev_priv->display.get_fifo_size = i830_get_fifo_size;
7998 }
7999 } else {
8000 drm_err(&dev_priv->drm,
8001 "unexpected fall-through in %s\n", __func__);
8002 }
8003}
8004
8005void intel_pm_setup(struct drm_i915_private *dev_priv)
8006{
8007 dev_priv->runtime_pm.suspended = false;
8008 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8009}
8010
8011static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8012{
8013 struct intel_dbuf_state *dbuf_state;
8014
8015 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8016 if (!dbuf_state)
8017 return NULL;
8018
8019 return &dbuf_state->base;
8020}
8021
8022static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8023 struct intel_global_state *state)
8024{
8025 kfree(state);
8026}
8027
8028static const struct intel_global_state_funcs intel_dbuf_funcs = {
8029 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8030 .atomic_destroy_state = intel_dbuf_destroy_state,
8031};
8032
8033struct intel_dbuf_state *
8034intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8035{
8036 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8037 struct intel_global_state *dbuf_state;
8038
8039 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8040 if (IS_ERR(dbuf_state))
8041 return ERR_CAST(dbuf_state);
8042
8043 return to_intel_dbuf_state(dbuf_state);
8044}
8045
8046int intel_dbuf_init(struct drm_i915_private *dev_priv)
8047{
8048 struct intel_dbuf_state *dbuf_state;
8049
8050 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8051 if (!dbuf_state)
8052 return -ENOMEM;
8053
8054 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8055 &dbuf_state->base, &intel_dbuf_funcs);
8056
8057 return 0;
8058}
8059
8060/*
8061 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8062 * update the request state of all DBUS slices.
8063 */
8064static void update_mbus_pre_enable(struct intel_atomic_state *state)
8065{
8066 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8067 u32 mbus_ctl, dbuf_min_tracker_val;
8068 enum dbuf_slice slice;
8069 const struct intel_dbuf_state *dbuf_state =
8070 intel_atomic_get_new_dbuf_state(state);
8071
8072 if (!IS_ALDERLAKE_P(dev_priv))
8073 return;
8074
8075 /*
8076 * TODO: Implement vblank synchronized MBUS joining changes.
8077 * Must be properly coordinated with dbuf reprogramming.
8078 */
8079 if (dbuf_state->joined_mbus) {
8080 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8081 MBUS_JOIN_PIPE_SELECT_NONE;
8082 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8083 } else {
8084 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8085 MBUS_JOIN_PIPE_SELECT_NONE;
8086 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8087 }
8088
8089 intel_de_rmw(dev_priv, MBUS_CTL,
8090 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8091 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8092
8093 for_each_dbuf_slice(dev_priv, slice)
8094 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8095 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8096 dbuf_min_tracker_val);
8097}
8098
8099void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8100{
8101 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8102 const struct intel_dbuf_state *new_dbuf_state =
8103 intel_atomic_get_new_dbuf_state(state);
8104 const struct intel_dbuf_state *old_dbuf_state =
8105 intel_atomic_get_old_dbuf_state(state);
8106
8107 if (!new_dbuf_state ||
8108 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8109 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8110 return;
8111
8112 WARN_ON(!new_dbuf_state->base.changed);
8113
8114 update_mbus_pre_enable(state);
8115 gen9_dbuf_slices_update(dev_priv,
8116 old_dbuf_state->enabled_slices |
8117 new_dbuf_state->enabled_slices);
8118}
8119
8120void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8121{
8122 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8123 const struct intel_dbuf_state *new_dbuf_state =
8124 intel_atomic_get_new_dbuf_state(state);
8125 const struct intel_dbuf_state *old_dbuf_state =
8126 intel_atomic_get_old_dbuf_state(state);
8127
8128 if (!new_dbuf_state ||
8129 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8130 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8131 return;
8132
8133 WARN_ON(!new_dbuf_state->base.changed);
8134
8135 gen9_dbuf_slices_update(dev_priv,
8136 new_dbuf_state->enabled_slices);
8137}
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
28#include <linux/cpufreq.h>
29#include "i915_drv.h"
30#include "intel_drv.h"
31#include "../../../platform/x86/intel_ips.h"
32#include <linux/module.h>
33#include <linux/vgaarb.h>
34#include <drm/i915_powerwell.h>
35#include <linux/pm_runtime.h>
36
37/**
38 * RC6 is a special power stage which allows the GPU to enter an very
39 * low-voltage mode when idle, using down to 0V while at this stage. This
40 * stage is entered automatically when the GPU is idle when RC6 support is
41 * enabled, and as soon as new workload arises GPU wakes up automatically as well.
42 *
43 * There are different RC6 modes available in Intel GPU, which differentiate
44 * among each other with the latency required to enter and leave RC6 and
45 * voltage consumed by the GPU in different states.
46 *
47 * The combination of the following flags define which states GPU is allowed
48 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
49 * RC6pp is deepest RC6. Their support by hardware varies according to the
50 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
51 * which brings the most power savings; deeper states save more power, but
52 * require higher latency to switch to and wake up.
53 */
54#define INTEL_RC6_ENABLE (1<<0)
55#define INTEL_RC6p_ENABLE (1<<1)
56#define INTEL_RC6pp_ENABLE (1<<2)
57
58/* FBC, or Frame Buffer Compression, is a technique employed to compress the
59 * framebuffer contents in-memory, aiming at reducing the required bandwidth
60 * during in-memory transfers and, therefore, reduce the power packet.
61 *
62 * The benefits of FBC are mostly visible with solid backgrounds and
63 * variation-less patterns.
64 *
65 * FBC-related functionality can be enabled by the means of the
66 * i915.i915_enable_fbc parameter
67 */
68
69static void i8xx_disable_fbc(struct drm_device *dev)
70{
71 struct drm_i915_private *dev_priv = dev->dev_private;
72 u32 fbc_ctl;
73
74 /* Disable compression */
75 fbc_ctl = I915_READ(FBC_CONTROL);
76 if ((fbc_ctl & FBC_CTL_EN) == 0)
77 return;
78
79 fbc_ctl &= ~FBC_CTL_EN;
80 I915_WRITE(FBC_CONTROL, fbc_ctl);
81
82 /* Wait for compressing bit to clear */
83 if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
84 DRM_DEBUG_KMS("FBC idle timed out\n");
85 return;
86 }
87
88 DRM_DEBUG_KMS("disabled FBC\n");
89}
90
91static void i8xx_enable_fbc(struct drm_crtc *crtc)
92{
93 struct drm_device *dev = crtc->dev;
94 struct drm_i915_private *dev_priv = dev->dev_private;
95 struct drm_framebuffer *fb = crtc->primary->fb;
96 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
97 struct drm_i915_gem_object *obj = intel_fb->obj;
98 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
99 int cfb_pitch;
100 int i;
101 u32 fbc_ctl;
102
103 cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
104 if (fb->pitches[0] < cfb_pitch)
105 cfb_pitch = fb->pitches[0];
106
107 /* FBC_CTL wants 32B or 64B units */
108 if (IS_GEN2(dev))
109 cfb_pitch = (cfb_pitch / 32) - 1;
110 else
111 cfb_pitch = (cfb_pitch / 64) - 1;
112
113 /* Clear old tags */
114 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
115 I915_WRITE(FBC_TAG + (i * 4), 0);
116
117 if (IS_GEN4(dev)) {
118 u32 fbc_ctl2;
119
120 /* Set it up... */
121 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
122 fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane);
123 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
124 I915_WRITE(FBC_FENCE_OFF, crtc->y);
125 }
126
127 /* enable it... */
128 fbc_ctl = I915_READ(FBC_CONTROL);
129 fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
130 fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC;
131 if (IS_I945GM(dev))
132 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
133 fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
134 fbc_ctl |= obj->fence_reg;
135 I915_WRITE(FBC_CONTROL, fbc_ctl);
136
137 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
138 cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
139}
140
141static bool i8xx_fbc_enabled(struct drm_device *dev)
142{
143 struct drm_i915_private *dev_priv = dev->dev_private;
144
145 return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
146}
147
148static void g4x_enable_fbc(struct drm_crtc *crtc)
149{
150 struct drm_device *dev = crtc->dev;
151 struct drm_i915_private *dev_priv = dev->dev_private;
152 struct drm_framebuffer *fb = crtc->primary->fb;
153 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
154 struct drm_i915_gem_object *obj = intel_fb->obj;
155 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
156 u32 dpfc_ctl;
157
158 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN;
159 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
160 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
161 else
162 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
163 dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
164
165 I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
166
167 /* enable it... */
168 I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
169
170 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
171}
172
173static void g4x_disable_fbc(struct drm_device *dev)
174{
175 struct drm_i915_private *dev_priv = dev->dev_private;
176 u32 dpfc_ctl;
177
178 /* Disable compression */
179 dpfc_ctl = I915_READ(DPFC_CONTROL);
180 if (dpfc_ctl & DPFC_CTL_EN) {
181 dpfc_ctl &= ~DPFC_CTL_EN;
182 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
183
184 DRM_DEBUG_KMS("disabled FBC\n");
185 }
186}
187
188static bool g4x_fbc_enabled(struct drm_device *dev)
189{
190 struct drm_i915_private *dev_priv = dev->dev_private;
191
192 return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
193}
194
195static void sandybridge_blit_fbc_update(struct drm_device *dev)
196{
197 struct drm_i915_private *dev_priv = dev->dev_private;
198 u32 blt_ecoskpd;
199
200 /* Make sure blitter notifies FBC of writes */
201
202 /* Blitter is part of Media powerwell on VLV. No impact of
203 * his param in other platforms for now */
204 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
205
206 blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
207 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
208 GEN6_BLITTER_LOCK_SHIFT;
209 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
210 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
211 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
212 blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
213 GEN6_BLITTER_LOCK_SHIFT);
214 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
215 POSTING_READ(GEN6_BLITTER_ECOSKPD);
216
217 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
218}
219
220static void ironlake_enable_fbc(struct drm_crtc *crtc)
221{
222 struct drm_device *dev = crtc->dev;
223 struct drm_i915_private *dev_priv = dev->dev_private;
224 struct drm_framebuffer *fb = crtc->primary->fb;
225 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
226 struct drm_i915_gem_object *obj = intel_fb->obj;
227 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
228 u32 dpfc_ctl;
229
230 dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane);
231 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
232 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
233 else
234 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
235 dpfc_ctl |= DPFC_CTL_FENCE_EN;
236 if (IS_GEN5(dev))
237 dpfc_ctl |= obj->fence_reg;
238
239 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
240 I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
241 /* enable it... */
242 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
243
244 if (IS_GEN6(dev)) {
245 I915_WRITE(SNB_DPFC_CTL_SA,
246 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
247 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
248 sandybridge_blit_fbc_update(dev);
249 }
250
251 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
252}
253
254static void ironlake_disable_fbc(struct drm_device *dev)
255{
256 struct drm_i915_private *dev_priv = dev->dev_private;
257 u32 dpfc_ctl;
258
259 /* Disable compression */
260 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
261 if (dpfc_ctl & DPFC_CTL_EN) {
262 dpfc_ctl &= ~DPFC_CTL_EN;
263 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
264
265 DRM_DEBUG_KMS("disabled FBC\n");
266 }
267}
268
269static bool ironlake_fbc_enabled(struct drm_device *dev)
270{
271 struct drm_i915_private *dev_priv = dev->dev_private;
272
273 return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
274}
275
276static void gen7_enable_fbc(struct drm_crtc *crtc)
277{
278 struct drm_device *dev = crtc->dev;
279 struct drm_i915_private *dev_priv = dev->dev_private;
280 struct drm_framebuffer *fb = crtc->primary->fb;
281 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
282 struct drm_i915_gem_object *obj = intel_fb->obj;
283 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
284 u32 dpfc_ctl;
285
286 dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane);
287 if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
288 dpfc_ctl |= DPFC_CTL_LIMIT_2X;
289 else
290 dpfc_ctl |= DPFC_CTL_LIMIT_1X;
291 dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
292
293 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
294
295 if (IS_IVYBRIDGE(dev)) {
296 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
297 I915_WRITE(ILK_DISPLAY_CHICKEN1,
298 I915_READ(ILK_DISPLAY_CHICKEN1) |
299 ILK_FBCQ_DIS);
300 } else {
301 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
302 I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe),
303 I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) |
304 HSW_FBCQ_DIS);
305 }
306
307 I915_WRITE(SNB_DPFC_CTL_SA,
308 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
309 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
310
311 sandybridge_blit_fbc_update(dev);
312
313 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
314}
315
316bool intel_fbc_enabled(struct drm_device *dev)
317{
318 struct drm_i915_private *dev_priv = dev->dev_private;
319
320 if (!dev_priv->display.fbc_enabled)
321 return false;
322
323 return dev_priv->display.fbc_enabled(dev);
324}
325
326static void intel_fbc_work_fn(struct work_struct *__work)
327{
328 struct intel_fbc_work *work =
329 container_of(to_delayed_work(__work),
330 struct intel_fbc_work, work);
331 struct drm_device *dev = work->crtc->dev;
332 struct drm_i915_private *dev_priv = dev->dev_private;
333
334 mutex_lock(&dev->struct_mutex);
335 if (work == dev_priv->fbc.fbc_work) {
336 /* Double check that we haven't switched fb without cancelling
337 * the prior work.
338 */
339 if (work->crtc->primary->fb == work->fb) {
340 dev_priv->display.enable_fbc(work->crtc);
341
342 dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
343 dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id;
344 dev_priv->fbc.y = work->crtc->y;
345 }
346
347 dev_priv->fbc.fbc_work = NULL;
348 }
349 mutex_unlock(&dev->struct_mutex);
350
351 kfree(work);
352}
353
354static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
355{
356 if (dev_priv->fbc.fbc_work == NULL)
357 return;
358
359 DRM_DEBUG_KMS("cancelling pending FBC enable\n");
360
361 /* Synchronisation is provided by struct_mutex and checking of
362 * dev_priv->fbc.fbc_work, so we can perform the cancellation
363 * entirely asynchronously.
364 */
365 if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
366 /* tasklet was killed before being run, clean up */
367 kfree(dev_priv->fbc.fbc_work);
368
369 /* Mark the work as no longer wanted so that if it does
370 * wake-up (because the work was already running and waiting
371 * for our mutex), it will discover that is no longer
372 * necessary to run.
373 */
374 dev_priv->fbc.fbc_work = NULL;
375}
376
377static void intel_enable_fbc(struct drm_crtc *crtc)
378{
379 struct intel_fbc_work *work;
380 struct drm_device *dev = crtc->dev;
381 struct drm_i915_private *dev_priv = dev->dev_private;
382
383 if (!dev_priv->display.enable_fbc)
384 return;
385
386 intel_cancel_fbc_work(dev_priv);
387
388 work = kzalloc(sizeof(*work), GFP_KERNEL);
389 if (work == NULL) {
390 DRM_ERROR("Failed to allocate FBC work structure\n");
391 dev_priv->display.enable_fbc(crtc);
392 return;
393 }
394
395 work->crtc = crtc;
396 work->fb = crtc->primary->fb;
397 INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
398
399 dev_priv->fbc.fbc_work = work;
400
401 /* Delay the actual enabling to let pageflipping cease and the
402 * display to settle before starting the compression. Note that
403 * this delay also serves a second purpose: it allows for a
404 * vblank to pass after disabling the FBC before we attempt
405 * to modify the control registers.
406 *
407 * A more complicated solution would involve tracking vblanks
408 * following the termination of the page-flipping sequence
409 * and indeed performing the enable as a co-routine and not
410 * waiting synchronously upon the vblank.
411 *
412 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
413 */
414 schedule_delayed_work(&work->work, msecs_to_jiffies(50));
415}
416
417void intel_disable_fbc(struct drm_device *dev)
418{
419 struct drm_i915_private *dev_priv = dev->dev_private;
420
421 intel_cancel_fbc_work(dev_priv);
422
423 if (!dev_priv->display.disable_fbc)
424 return;
425
426 dev_priv->display.disable_fbc(dev);
427 dev_priv->fbc.plane = -1;
428}
429
430static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
431 enum no_fbc_reason reason)
432{
433 if (dev_priv->fbc.no_fbc_reason == reason)
434 return false;
435
436 dev_priv->fbc.no_fbc_reason = reason;
437 return true;
438}
439
440/**
441 * intel_update_fbc - enable/disable FBC as needed
442 * @dev: the drm_device
443 *
444 * Set up the framebuffer compression hardware at mode set time. We
445 * enable it if possible:
446 * - plane A only (on pre-965)
447 * - no pixel mulitply/line duplication
448 * - no alpha buffer discard
449 * - no dual wide
450 * - framebuffer <= max_hdisplay in width, max_vdisplay in height
451 *
452 * We can't assume that any compression will take place (worst case),
453 * so the compressed buffer has to be the same size as the uncompressed
454 * one. It also must reside (along with the line length buffer) in
455 * stolen memory.
456 *
457 * We need to enable/disable FBC on a global basis.
458 */
459void intel_update_fbc(struct drm_device *dev)
460{
461 struct drm_i915_private *dev_priv = dev->dev_private;
462 struct drm_crtc *crtc = NULL, *tmp_crtc;
463 struct intel_crtc *intel_crtc;
464 struct drm_framebuffer *fb;
465 struct intel_framebuffer *intel_fb;
466 struct drm_i915_gem_object *obj;
467 const struct drm_display_mode *adjusted_mode;
468 unsigned int max_width, max_height;
469
470 if (!HAS_FBC(dev)) {
471 set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
472 return;
473 }
474
475 if (!i915.powersave) {
476 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
477 DRM_DEBUG_KMS("fbc disabled per module param\n");
478 return;
479 }
480
481 /*
482 * If FBC is already on, we just have to verify that we can
483 * keep it that way...
484 * Need to disable if:
485 * - more than one pipe is active
486 * - changing FBC params (stride, fence, mode)
487 * - new fb is too large to fit in compressed buffer
488 * - going to an unsupported config (interlace, pixel multiply, etc.)
489 */
490 list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
491 if (intel_crtc_active(tmp_crtc) &&
492 to_intel_crtc(tmp_crtc)->primary_enabled) {
493 if (crtc) {
494 if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
495 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
496 goto out_disable;
497 }
498 crtc = tmp_crtc;
499 }
500 }
501
502 if (!crtc || crtc->primary->fb == NULL) {
503 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
504 DRM_DEBUG_KMS("no output, disabling\n");
505 goto out_disable;
506 }
507
508 intel_crtc = to_intel_crtc(crtc);
509 fb = crtc->primary->fb;
510 intel_fb = to_intel_framebuffer(fb);
511 obj = intel_fb->obj;
512 adjusted_mode = &intel_crtc->config.adjusted_mode;
513
514 if (i915.enable_fbc < 0 &&
515 INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
516 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
517 DRM_DEBUG_KMS("disabled per chip default\n");
518 goto out_disable;
519 }
520 if (!i915.enable_fbc) {
521 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
522 DRM_DEBUG_KMS("fbc disabled per module param\n");
523 goto out_disable;
524 }
525 if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ||
526 (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
527 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
528 DRM_DEBUG_KMS("mode incompatible with compression, "
529 "disabling\n");
530 goto out_disable;
531 }
532
533 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
534 max_width = 4096;
535 max_height = 2048;
536 } else {
537 max_width = 2048;
538 max_height = 1536;
539 }
540 if (intel_crtc->config.pipe_src_w > max_width ||
541 intel_crtc->config.pipe_src_h > max_height) {
542 if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
543 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
544 goto out_disable;
545 }
546 if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) &&
547 intel_crtc->plane != PLANE_A) {
548 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
549 DRM_DEBUG_KMS("plane not A, disabling compression\n");
550 goto out_disable;
551 }
552
553 /* The use of a CPU fence is mandatory in order to detect writes
554 * by the CPU to the scanout and trigger updates to the FBC.
555 */
556 if (obj->tiling_mode != I915_TILING_X ||
557 obj->fence_reg == I915_FENCE_REG_NONE) {
558 if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
559 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
560 goto out_disable;
561 }
562
563 /* If the kernel debugger is active, always disable compression */
564 if (in_dbg_master())
565 goto out_disable;
566
567 if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
568 if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
569 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
570 goto out_disable;
571 }
572
573 /* If the scanout has not changed, don't modify the FBC settings.
574 * Note that we make the fundamental assumption that the fb->obj
575 * cannot be unpinned (and have its GTT offset and fence revoked)
576 * without first being decoupled from the scanout and FBC disabled.
577 */
578 if (dev_priv->fbc.plane == intel_crtc->plane &&
579 dev_priv->fbc.fb_id == fb->base.id &&
580 dev_priv->fbc.y == crtc->y)
581 return;
582
583 if (intel_fbc_enabled(dev)) {
584 /* We update FBC along two paths, after changing fb/crtc
585 * configuration (modeswitching) and after page-flipping
586 * finishes. For the latter, we know that not only did
587 * we disable the FBC at the start of the page-flip
588 * sequence, but also more than one vblank has passed.
589 *
590 * For the former case of modeswitching, it is possible
591 * to switch between two FBC valid configurations
592 * instantaneously so we do need to disable the FBC
593 * before we can modify its control registers. We also
594 * have to wait for the next vblank for that to take
595 * effect. However, since we delay enabling FBC we can
596 * assume that a vblank has passed since disabling and
597 * that we can safely alter the registers in the deferred
598 * callback.
599 *
600 * In the scenario that we go from a valid to invalid
601 * and then back to valid FBC configuration we have
602 * no strict enforcement that a vblank occurred since
603 * disabling the FBC. However, along all current pipe
604 * disabling paths we do need to wait for a vblank at
605 * some point. And we wait before enabling FBC anyway.
606 */
607 DRM_DEBUG_KMS("disabling active FBC for update\n");
608 intel_disable_fbc(dev);
609 }
610
611 intel_enable_fbc(crtc);
612 dev_priv->fbc.no_fbc_reason = FBC_OK;
613 return;
614
615out_disable:
616 /* Multiple disables should be harmless */
617 if (intel_fbc_enabled(dev)) {
618 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
619 intel_disable_fbc(dev);
620 }
621 i915_gem_stolen_cleanup_compression(dev);
622}
623
624static void i915_pineview_get_mem_freq(struct drm_device *dev)
625{
626 struct drm_i915_private *dev_priv = dev->dev_private;
627 u32 tmp;
628
629 tmp = I915_READ(CLKCFG);
630
631 switch (tmp & CLKCFG_FSB_MASK) {
632 case CLKCFG_FSB_533:
633 dev_priv->fsb_freq = 533; /* 133*4 */
634 break;
635 case CLKCFG_FSB_800:
636 dev_priv->fsb_freq = 800; /* 200*4 */
637 break;
638 case CLKCFG_FSB_667:
639 dev_priv->fsb_freq = 667; /* 167*4 */
640 break;
641 case CLKCFG_FSB_400:
642 dev_priv->fsb_freq = 400; /* 100*4 */
643 break;
644 }
645
646 switch (tmp & CLKCFG_MEM_MASK) {
647 case CLKCFG_MEM_533:
648 dev_priv->mem_freq = 533;
649 break;
650 case CLKCFG_MEM_667:
651 dev_priv->mem_freq = 667;
652 break;
653 case CLKCFG_MEM_800:
654 dev_priv->mem_freq = 800;
655 break;
656 }
657
658 /* detect pineview DDR3 setting */
659 tmp = I915_READ(CSHRDDR3CTL);
660 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
661}
662
663static void i915_ironlake_get_mem_freq(struct drm_device *dev)
664{
665 struct drm_i915_private *dev_priv = dev->dev_private;
666 u16 ddrpll, csipll;
667
668 ddrpll = I915_READ16(DDRMPLL1);
669 csipll = I915_READ16(CSIPLL0);
670
671 switch (ddrpll & 0xff) {
672 case 0xc:
673 dev_priv->mem_freq = 800;
674 break;
675 case 0x10:
676 dev_priv->mem_freq = 1066;
677 break;
678 case 0x14:
679 dev_priv->mem_freq = 1333;
680 break;
681 case 0x18:
682 dev_priv->mem_freq = 1600;
683 break;
684 default:
685 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
686 ddrpll & 0xff);
687 dev_priv->mem_freq = 0;
688 break;
689 }
690
691 dev_priv->ips.r_t = dev_priv->mem_freq;
692
693 switch (csipll & 0x3ff) {
694 case 0x00c:
695 dev_priv->fsb_freq = 3200;
696 break;
697 case 0x00e:
698 dev_priv->fsb_freq = 3733;
699 break;
700 case 0x010:
701 dev_priv->fsb_freq = 4266;
702 break;
703 case 0x012:
704 dev_priv->fsb_freq = 4800;
705 break;
706 case 0x014:
707 dev_priv->fsb_freq = 5333;
708 break;
709 case 0x016:
710 dev_priv->fsb_freq = 5866;
711 break;
712 case 0x018:
713 dev_priv->fsb_freq = 6400;
714 break;
715 default:
716 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
717 csipll & 0x3ff);
718 dev_priv->fsb_freq = 0;
719 break;
720 }
721
722 if (dev_priv->fsb_freq == 3200) {
723 dev_priv->ips.c_m = 0;
724 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
725 dev_priv->ips.c_m = 1;
726 } else {
727 dev_priv->ips.c_m = 2;
728 }
729}
730
731static const struct cxsr_latency cxsr_latency_table[] = {
732 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
733 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
734 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
735 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
736 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
737
738 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
739 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
740 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
741 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
742 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
743
744 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
745 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
746 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
747 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
748 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
749
750 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
751 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
752 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
753 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
754 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
755
756 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
757 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
758 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
759 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
760 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
761
762 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
763 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
764 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
765 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
766 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
767};
768
769static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
770 int is_ddr3,
771 int fsb,
772 int mem)
773{
774 const struct cxsr_latency *latency;
775 int i;
776
777 if (fsb == 0 || mem == 0)
778 return NULL;
779
780 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
781 latency = &cxsr_latency_table[i];
782 if (is_desktop == latency->is_desktop &&
783 is_ddr3 == latency->is_ddr3 &&
784 fsb == latency->fsb_freq && mem == latency->mem_freq)
785 return latency;
786 }
787
788 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
789
790 return NULL;
791}
792
793static void pineview_disable_cxsr(struct drm_device *dev)
794{
795 struct drm_i915_private *dev_priv = dev->dev_private;
796
797 /* deactivate cxsr */
798 I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
799}
800
801/*
802 * Latency for FIFO fetches is dependent on several factors:
803 * - memory configuration (speed, channels)
804 * - chipset
805 * - current MCH state
806 * It can be fairly high in some situations, so here we assume a fairly
807 * pessimal value. It's a tradeoff between extra memory fetches (if we
808 * set this value too high, the FIFO will fetch frequently to stay full)
809 * and power consumption (set it too low to save power and we might see
810 * FIFO underruns and display "flicker").
811 *
812 * A value of 5us seems to be a good balance; safe for very low end
813 * platforms but not overly aggressive on lower latency configs.
814 */
815static const int latency_ns = 5000;
816
817static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
818{
819 struct drm_i915_private *dev_priv = dev->dev_private;
820 uint32_t dsparb = I915_READ(DSPARB);
821 int size;
822
823 size = dsparb & 0x7f;
824 if (plane)
825 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
826
827 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
828 plane ? "B" : "A", size);
829
830 return size;
831}
832
833static int i830_get_fifo_size(struct drm_device *dev, int plane)
834{
835 struct drm_i915_private *dev_priv = dev->dev_private;
836 uint32_t dsparb = I915_READ(DSPARB);
837 int size;
838
839 size = dsparb & 0x1ff;
840 if (plane)
841 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
842 size >>= 1; /* Convert to cachelines */
843
844 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
845 plane ? "B" : "A", size);
846
847 return size;
848}
849
850static int i845_get_fifo_size(struct drm_device *dev, int plane)
851{
852 struct drm_i915_private *dev_priv = dev->dev_private;
853 uint32_t dsparb = I915_READ(DSPARB);
854 int size;
855
856 size = dsparb & 0x7f;
857 size >>= 2; /* Convert to cachelines */
858
859 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
860 plane ? "B" : "A",
861 size);
862
863 return size;
864}
865
866/* Pineview has different values for various configs */
867static const struct intel_watermark_params pineview_display_wm = {
868 PINEVIEW_DISPLAY_FIFO,
869 PINEVIEW_MAX_WM,
870 PINEVIEW_DFT_WM,
871 PINEVIEW_GUARD_WM,
872 PINEVIEW_FIFO_LINE_SIZE
873};
874static const struct intel_watermark_params pineview_display_hplloff_wm = {
875 PINEVIEW_DISPLAY_FIFO,
876 PINEVIEW_MAX_WM,
877 PINEVIEW_DFT_HPLLOFF_WM,
878 PINEVIEW_GUARD_WM,
879 PINEVIEW_FIFO_LINE_SIZE
880};
881static const struct intel_watermark_params pineview_cursor_wm = {
882 PINEVIEW_CURSOR_FIFO,
883 PINEVIEW_CURSOR_MAX_WM,
884 PINEVIEW_CURSOR_DFT_WM,
885 PINEVIEW_CURSOR_GUARD_WM,
886 PINEVIEW_FIFO_LINE_SIZE,
887};
888static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
889 PINEVIEW_CURSOR_FIFO,
890 PINEVIEW_CURSOR_MAX_WM,
891 PINEVIEW_CURSOR_DFT_WM,
892 PINEVIEW_CURSOR_GUARD_WM,
893 PINEVIEW_FIFO_LINE_SIZE
894};
895static const struct intel_watermark_params g4x_wm_info = {
896 G4X_FIFO_SIZE,
897 G4X_MAX_WM,
898 G4X_MAX_WM,
899 2,
900 G4X_FIFO_LINE_SIZE,
901};
902static const struct intel_watermark_params g4x_cursor_wm_info = {
903 I965_CURSOR_FIFO,
904 I965_CURSOR_MAX_WM,
905 I965_CURSOR_DFT_WM,
906 2,
907 G4X_FIFO_LINE_SIZE,
908};
909static const struct intel_watermark_params valleyview_wm_info = {
910 VALLEYVIEW_FIFO_SIZE,
911 VALLEYVIEW_MAX_WM,
912 VALLEYVIEW_MAX_WM,
913 2,
914 G4X_FIFO_LINE_SIZE,
915};
916static const struct intel_watermark_params valleyview_cursor_wm_info = {
917 I965_CURSOR_FIFO,
918 VALLEYVIEW_CURSOR_MAX_WM,
919 I965_CURSOR_DFT_WM,
920 2,
921 G4X_FIFO_LINE_SIZE,
922};
923static const struct intel_watermark_params i965_cursor_wm_info = {
924 I965_CURSOR_FIFO,
925 I965_CURSOR_MAX_WM,
926 I965_CURSOR_DFT_WM,
927 2,
928 I915_FIFO_LINE_SIZE,
929};
930static const struct intel_watermark_params i945_wm_info = {
931 I945_FIFO_SIZE,
932 I915_MAX_WM,
933 1,
934 2,
935 I915_FIFO_LINE_SIZE
936};
937static const struct intel_watermark_params i915_wm_info = {
938 I915_FIFO_SIZE,
939 I915_MAX_WM,
940 1,
941 2,
942 I915_FIFO_LINE_SIZE
943};
944static const struct intel_watermark_params i830_wm_info = {
945 I855GM_FIFO_SIZE,
946 I915_MAX_WM,
947 1,
948 2,
949 I830_FIFO_LINE_SIZE
950};
951static const struct intel_watermark_params i845_wm_info = {
952 I830_FIFO_SIZE,
953 I915_MAX_WM,
954 1,
955 2,
956 I830_FIFO_LINE_SIZE
957};
958
959/**
960 * intel_calculate_wm - calculate watermark level
961 * @clock_in_khz: pixel clock
962 * @wm: chip FIFO params
963 * @pixel_size: display pixel size
964 * @latency_ns: memory latency for the platform
965 *
966 * Calculate the watermark level (the level at which the display plane will
967 * start fetching from memory again). Each chip has a different display
968 * FIFO size and allocation, so the caller needs to figure that out and pass
969 * in the correct intel_watermark_params structure.
970 *
971 * As the pixel clock runs, the FIFO will be drained at a rate that depends
972 * on the pixel size. When it reaches the watermark level, it'll start
973 * fetching FIFO line sized based chunks from memory until the FIFO fills
974 * past the watermark point. If the FIFO drains completely, a FIFO underrun
975 * will occur, and a display engine hang could result.
976 */
977static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
978 const struct intel_watermark_params *wm,
979 int fifo_size,
980 int pixel_size,
981 unsigned long latency_ns)
982{
983 long entries_required, wm_size;
984
985 /*
986 * Note: we need to make sure we don't overflow for various clock &
987 * latency values.
988 * clocks go from a few thousand to several hundred thousand.
989 * latency is usually a few thousand
990 */
991 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
992 1000;
993 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
994
995 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
996
997 wm_size = fifo_size - (entries_required + wm->guard_size);
998
999 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
1000
1001 /* Don't promote wm_size to unsigned... */
1002 if (wm_size > (long)wm->max_wm)
1003 wm_size = wm->max_wm;
1004 if (wm_size <= 0)
1005 wm_size = wm->default_wm;
1006 return wm_size;
1007}
1008
1009static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
1010{
1011 struct drm_crtc *crtc, *enabled = NULL;
1012
1013 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1014 if (intel_crtc_active(crtc)) {
1015 if (enabled)
1016 return NULL;
1017 enabled = crtc;
1018 }
1019 }
1020
1021 return enabled;
1022}
1023
1024static void pineview_update_wm(struct drm_crtc *unused_crtc)
1025{
1026 struct drm_device *dev = unused_crtc->dev;
1027 struct drm_i915_private *dev_priv = dev->dev_private;
1028 struct drm_crtc *crtc;
1029 const struct cxsr_latency *latency;
1030 u32 reg;
1031 unsigned long wm;
1032
1033 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
1034 dev_priv->fsb_freq, dev_priv->mem_freq);
1035 if (!latency) {
1036 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
1037 pineview_disable_cxsr(dev);
1038 return;
1039 }
1040
1041 crtc = single_enabled_crtc(dev);
1042 if (crtc) {
1043 const struct drm_display_mode *adjusted_mode;
1044 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
1045 int clock;
1046
1047 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1048 clock = adjusted_mode->crtc_clock;
1049
1050 /* Display SR */
1051 wm = intel_calculate_wm(clock, &pineview_display_wm,
1052 pineview_display_wm.fifo_size,
1053 pixel_size, latency->display_sr);
1054 reg = I915_READ(DSPFW1);
1055 reg &= ~DSPFW_SR_MASK;
1056 reg |= wm << DSPFW_SR_SHIFT;
1057 I915_WRITE(DSPFW1, reg);
1058 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
1059
1060 /* cursor SR */
1061 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
1062 pineview_display_wm.fifo_size,
1063 pixel_size, latency->cursor_sr);
1064 reg = I915_READ(DSPFW3);
1065 reg &= ~DSPFW_CURSOR_SR_MASK;
1066 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
1067 I915_WRITE(DSPFW3, reg);
1068
1069 /* Display HPLL off SR */
1070 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
1071 pineview_display_hplloff_wm.fifo_size,
1072 pixel_size, latency->display_hpll_disable);
1073 reg = I915_READ(DSPFW3);
1074 reg &= ~DSPFW_HPLL_SR_MASK;
1075 reg |= wm & DSPFW_HPLL_SR_MASK;
1076 I915_WRITE(DSPFW3, reg);
1077
1078 /* cursor HPLL off SR */
1079 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
1080 pineview_display_hplloff_wm.fifo_size,
1081 pixel_size, latency->cursor_hpll_disable);
1082 reg = I915_READ(DSPFW3);
1083 reg &= ~DSPFW_HPLL_CURSOR_MASK;
1084 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
1085 I915_WRITE(DSPFW3, reg);
1086 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
1087
1088 /* activate cxsr */
1089 I915_WRITE(DSPFW3,
1090 I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
1091 DRM_DEBUG_KMS("Self-refresh is enabled\n");
1092 } else {
1093 pineview_disable_cxsr(dev);
1094 DRM_DEBUG_KMS("Self-refresh is disabled\n");
1095 }
1096}
1097
1098static bool g4x_compute_wm0(struct drm_device *dev,
1099 int plane,
1100 const struct intel_watermark_params *display,
1101 int display_latency_ns,
1102 const struct intel_watermark_params *cursor,
1103 int cursor_latency_ns,
1104 int *plane_wm,
1105 int *cursor_wm)
1106{
1107 struct drm_crtc *crtc;
1108 const struct drm_display_mode *adjusted_mode;
1109 int htotal, hdisplay, clock, pixel_size;
1110 int line_time_us, line_count;
1111 int entries, tlb_miss;
1112
1113 crtc = intel_get_crtc_for_plane(dev, plane);
1114 if (!intel_crtc_active(crtc)) {
1115 *cursor_wm = cursor->guard_size;
1116 *plane_wm = display->guard_size;
1117 return false;
1118 }
1119
1120 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1121 clock = adjusted_mode->crtc_clock;
1122 htotal = adjusted_mode->crtc_htotal;
1123 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1124 pixel_size = crtc->primary->fb->bits_per_pixel / 8;
1125
1126 /* Use the small buffer method to calculate plane watermark */
1127 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
1128 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
1129 if (tlb_miss > 0)
1130 entries += tlb_miss;
1131 entries = DIV_ROUND_UP(entries, display->cacheline_size);
1132 *plane_wm = entries + display->guard_size;
1133 if (*plane_wm > (int)display->max_wm)
1134 *plane_wm = display->max_wm;
1135
1136 /* Use the large buffer method to calculate cursor watermark */
1137 line_time_us = max(htotal * 1000 / clock, 1);
1138 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
1139 entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
1140 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
1141 if (tlb_miss > 0)
1142 entries += tlb_miss;
1143 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1144 *cursor_wm = entries + cursor->guard_size;
1145 if (*cursor_wm > (int)cursor->max_wm)
1146 *cursor_wm = (int)cursor->max_wm;
1147
1148 return true;
1149}
1150
1151/*
1152 * Check the wm result.
1153 *
1154 * If any calculated watermark values is larger than the maximum value that
1155 * can be programmed into the associated watermark register, that watermark
1156 * must be disabled.
1157 */
1158static bool g4x_check_srwm(struct drm_device *dev,
1159 int display_wm, int cursor_wm,
1160 const struct intel_watermark_params *display,
1161 const struct intel_watermark_params *cursor)
1162{
1163 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1164 display_wm, cursor_wm);
1165
1166 if (display_wm > display->max_wm) {
1167 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1168 display_wm, display->max_wm);
1169 return false;
1170 }
1171
1172 if (cursor_wm > cursor->max_wm) {
1173 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1174 cursor_wm, cursor->max_wm);
1175 return false;
1176 }
1177
1178 if (!(display_wm || cursor_wm)) {
1179 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1180 return false;
1181 }
1182
1183 return true;
1184}
1185
1186static bool g4x_compute_srwm(struct drm_device *dev,
1187 int plane,
1188 int latency_ns,
1189 const struct intel_watermark_params *display,
1190 const struct intel_watermark_params *cursor,
1191 int *display_wm, int *cursor_wm)
1192{
1193 struct drm_crtc *crtc;
1194 const struct drm_display_mode *adjusted_mode;
1195 int hdisplay, htotal, pixel_size, clock;
1196 unsigned long line_time_us;
1197 int line_count, line_size;
1198 int small, large;
1199 int entries;
1200
1201 if (!latency_ns) {
1202 *display_wm = *cursor_wm = 0;
1203 return false;
1204 }
1205
1206 crtc = intel_get_crtc_for_plane(dev, plane);
1207 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1208 clock = adjusted_mode->crtc_clock;
1209 htotal = adjusted_mode->crtc_htotal;
1210 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1211 pixel_size = crtc->primary->fb->bits_per_pixel / 8;
1212
1213 line_time_us = max(htotal * 1000 / clock, 1);
1214 line_count = (latency_ns / line_time_us + 1000) / 1000;
1215 line_size = hdisplay * pixel_size;
1216
1217 /* Use the minimum of the small and large buffer method for primary */
1218 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1219 large = line_count * line_size;
1220
1221 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1222 *display_wm = entries + display->guard_size;
1223
1224 /* calculate the self-refresh watermark for display cursor */
1225 entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
1226 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1227 *cursor_wm = entries + cursor->guard_size;
1228
1229 return g4x_check_srwm(dev,
1230 *display_wm, *cursor_wm,
1231 display, cursor);
1232}
1233
1234static bool vlv_compute_drain_latency(struct drm_device *dev,
1235 int plane,
1236 int *plane_prec_mult,
1237 int *plane_dl,
1238 int *cursor_prec_mult,
1239 int *cursor_dl)
1240{
1241 struct drm_crtc *crtc;
1242 int clock, pixel_size;
1243 int entries;
1244
1245 crtc = intel_get_crtc_for_plane(dev, plane);
1246 if (!intel_crtc_active(crtc))
1247 return false;
1248
1249 clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
1250 pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
1251
1252 entries = (clock / 1000) * pixel_size;
1253 *plane_prec_mult = (entries > 256) ?
1254 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1255 *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
1256 pixel_size);
1257
1258 entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
1259 *cursor_prec_mult = (entries > 256) ?
1260 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1261 *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
1262
1263 return true;
1264}
1265
1266/*
1267 * Update drain latency registers of memory arbiter
1268 *
1269 * Valleyview SoC has a new memory arbiter and needs drain latency registers
1270 * to be programmed. Each plane has a drain latency multiplier and a drain
1271 * latency value.
1272 */
1273
1274static void vlv_update_drain_latency(struct drm_device *dev)
1275{
1276 struct drm_i915_private *dev_priv = dev->dev_private;
1277 int planea_prec, planea_dl, planeb_prec, planeb_dl;
1278 int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
1279 int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
1280 either 16 or 32 */
1281
1282 /* For plane A, Cursor A */
1283 if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
1284 &cursor_prec_mult, &cursora_dl)) {
1285 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1286 DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
1287 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1288 DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
1289
1290 I915_WRITE(VLV_DDL1, cursora_prec |
1291 (cursora_dl << DDL_CURSORA_SHIFT) |
1292 planea_prec | planea_dl);
1293 }
1294
1295 /* For plane B, Cursor B */
1296 if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
1297 &cursor_prec_mult, &cursorb_dl)) {
1298 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1299 DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
1300 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1301 DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
1302
1303 I915_WRITE(VLV_DDL2, cursorb_prec |
1304 (cursorb_dl << DDL_CURSORB_SHIFT) |
1305 planeb_prec | planeb_dl);
1306 }
1307}
1308
1309#define single_plane_enabled(mask) is_power_of_2(mask)
1310
1311static void valleyview_update_wm(struct drm_crtc *crtc)
1312{
1313 struct drm_device *dev = crtc->dev;
1314 static const int sr_latency_ns = 12000;
1315 struct drm_i915_private *dev_priv = dev->dev_private;
1316 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1317 int plane_sr, cursor_sr;
1318 int ignore_plane_sr, ignore_cursor_sr;
1319 unsigned int enabled = 0;
1320
1321 vlv_update_drain_latency(dev);
1322
1323 if (g4x_compute_wm0(dev, PIPE_A,
1324 &valleyview_wm_info, latency_ns,
1325 &valleyview_cursor_wm_info, latency_ns,
1326 &planea_wm, &cursora_wm))
1327 enabled |= 1 << PIPE_A;
1328
1329 if (g4x_compute_wm0(dev, PIPE_B,
1330 &valleyview_wm_info, latency_ns,
1331 &valleyview_cursor_wm_info, latency_ns,
1332 &planeb_wm, &cursorb_wm))
1333 enabled |= 1 << PIPE_B;
1334
1335 if (single_plane_enabled(enabled) &&
1336 g4x_compute_srwm(dev, ffs(enabled) - 1,
1337 sr_latency_ns,
1338 &valleyview_wm_info,
1339 &valleyview_cursor_wm_info,
1340 &plane_sr, &ignore_cursor_sr) &&
1341 g4x_compute_srwm(dev, ffs(enabled) - 1,
1342 2*sr_latency_ns,
1343 &valleyview_wm_info,
1344 &valleyview_cursor_wm_info,
1345 &ignore_plane_sr, &cursor_sr)) {
1346 I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
1347 } else {
1348 I915_WRITE(FW_BLC_SELF_VLV,
1349 I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
1350 plane_sr = cursor_sr = 0;
1351 }
1352
1353 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1354 planea_wm, cursora_wm,
1355 planeb_wm, cursorb_wm,
1356 plane_sr, cursor_sr);
1357
1358 I915_WRITE(DSPFW1,
1359 (plane_sr << DSPFW_SR_SHIFT) |
1360 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1361 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1362 planea_wm);
1363 I915_WRITE(DSPFW2,
1364 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1365 (cursora_wm << DSPFW_CURSORA_SHIFT));
1366 I915_WRITE(DSPFW3,
1367 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
1368 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1369}
1370
1371static void g4x_update_wm(struct drm_crtc *crtc)
1372{
1373 struct drm_device *dev = crtc->dev;
1374 static const int sr_latency_ns = 12000;
1375 struct drm_i915_private *dev_priv = dev->dev_private;
1376 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1377 int plane_sr, cursor_sr;
1378 unsigned int enabled = 0;
1379
1380 if (g4x_compute_wm0(dev, PIPE_A,
1381 &g4x_wm_info, latency_ns,
1382 &g4x_cursor_wm_info, latency_ns,
1383 &planea_wm, &cursora_wm))
1384 enabled |= 1 << PIPE_A;
1385
1386 if (g4x_compute_wm0(dev, PIPE_B,
1387 &g4x_wm_info, latency_ns,
1388 &g4x_cursor_wm_info, latency_ns,
1389 &planeb_wm, &cursorb_wm))
1390 enabled |= 1 << PIPE_B;
1391
1392 if (single_plane_enabled(enabled) &&
1393 g4x_compute_srwm(dev, ffs(enabled) - 1,
1394 sr_latency_ns,
1395 &g4x_wm_info,
1396 &g4x_cursor_wm_info,
1397 &plane_sr, &cursor_sr)) {
1398 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1399 } else {
1400 I915_WRITE(FW_BLC_SELF,
1401 I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
1402 plane_sr = cursor_sr = 0;
1403 }
1404
1405 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1406 planea_wm, cursora_wm,
1407 planeb_wm, cursorb_wm,
1408 plane_sr, cursor_sr);
1409
1410 I915_WRITE(DSPFW1,
1411 (plane_sr << DSPFW_SR_SHIFT) |
1412 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1413 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1414 planea_wm);
1415 I915_WRITE(DSPFW2,
1416 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1417 (cursora_wm << DSPFW_CURSORA_SHIFT));
1418 /* HPLL off in SR has some issues on G4x... disable it */
1419 I915_WRITE(DSPFW3,
1420 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1421 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1422}
1423
1424static void i965_update_wm(struct drm_crtc *unused_crtc)
1425{
1426 struct drm_device *dev = unused_crtc->dev;
1427 struct drm_i915_private *dev_priv = dev->dev_private;
1428 struct drm_crtc *crtc;
1429 int srwm = 1;
1430 int cursor_sr = 16;
1431
1432 /* Calc sr entries for one plane configs */
1433 crtc = single_enabled_crtc(dev);
1434 if (crtc) {
1435 /* self-refresh has much higher latency */
1436 static const int sr_latency_ns = 12000;
1437 const struct drm_display_mode *adjusted_mode =
1438 &to_intel_crtc(crtc)->config.adjusted_mode;
1439 int clock = adjusted_mode->crtc_clock;
1440 int htotal = adjusted_mode->crtc_htotal;
1441 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1442 int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
1443 unsigned long line_time_us;
1444 int entries;
1445
1446 line_time_us = max(htotal * 1000 / clock, 1);
1447
1448 /* Use ns/us then divide to preserve precision */
1449 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1450 pixel_size * hdisplay;
1451 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1452 srwm = I965_FIFO_SIZE - entries;
1453 if (srwm < 0)
1454 srwm = 1;
1455 srwm &= 0x1ff;
1456 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1457 entries, srwm);
1458
1459 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1460 pixel_size * to_intel_crtc(crtc)->cursor_width;
1461 entries = DIV_ROUND_UP(entries,
1462 i965_cursor_wm_info.cacheline_size);
1463 cursor_sr = i965_cursor_wm_info.fifo_size -
1464 (entries + i965_cursor_wm_info.guard_size);
1465
1466 if (cursor_sr > i965_cursor_wm_info.max_wm)
1467 cursor_sr = i965_cursor_wm_info.max_wm;
1468
1469 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1470 "cursor %d\n", srwm, cursor_sr);
1471
1472 if (IS_CRESTLINE(dev))
1473 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1474 } else {
1475 /* Turn off self refresh if both pipes are enabled */
1476 if (IS_CRESTLINE(dev))
1477 I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
1478 & ~FW_BLC_SELF_EN);
1479 }
1480
1481 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1482 srwm);
1483
1484 /* 965 has limitations... */
1485 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1486 (8 << 16) | (8 << 8) | (8 << 0));
1487 I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
1488 /* update cursor SR watermark */
1489 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1490}
1491
1492static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1493{
1494 struct drm_device *dev = unused_crtc->dev;
1495 struct drm_i915_private *dev_priv = dev->dev_private;
1496 const struct intel_watermark_params *wm_info;
1497 uint32_t fwater_lo;
1498 uint32_t fwater_hi;
1499 int cwm, srwm = 1;
1500 int fifo_size;
1501 int planea_wm, planeb_wm;
1502 struct drm_crtc *crtc, *enabled = NULL;
1503
1504 if (IS_I945GM(dev))
1505 wm_info = &i945_wm_info;
1506 else if (!IS_GEN2(dev))
1507 wm_info = &i915_wm_info;
1508 else
1509 wm_info = &i830_wm_info;
1510
1511 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1512 crtc = intel_get_crtc_for_plane(dev, 0);
1513 if (intel_crtc_active(crtc)) {
1514 const struct drm_display_mode *adjusted_mode;
1515 int cpp = crtc->primary->fb->bits_per_pixel / 8;
1516 if (IS_GEN2(dev))
1517 cpp = 4;
1518
1519 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1520 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1521 wm_info, fifo_size, cpp,
1522 latency_ns);
1523 enabled = crtc;
1524 } else
1525 planea_wm = fifo_size - wm_info->guard_size;
1526
1527 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1528 crtc = intel_get_crtc_for_plane(dev, 1);
1529 if (intel_crtc_active(crtc)) {
1530 const struct drm_display_mode *adjusted_mode;
1531 int cpp = crtc->primary->fb->bits_per_pixel / 8;
1532 if (IS_GEN2(dev))
1533 cpp = 4;
1534
1535 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1536 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1537 wm_info, fifo_size, cpp,
1538 latency_ns);
1539 if (enabled == NULL)
1540 enabled = crtc;
1541 else
1542 enabled = NULL;
1543 } else
1544 planeb_wm = fifo_size - wm_info->guard_size;
1545
1546 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1547
1548 if (IS_I915GM(dev) && enabled) {
1549 struct intel_framebuffer *fb;
1550
1551 fb = to_intel_framebuffer(enabled->primary->fb);
1552
1553 /* self-refresh seems busted with untiled */
1554 if (fb->obj->tiling_mode == I915_TILING_NONE)
1555 enabled = NULL;
1556 }
1557
1558 /*
1559 * Overlay gets an aggressive default since video jitter is bad.
1560 */
1561 cwm = 2;
1562
1563 /* Play safe and disable self-refresh before adjusting watermarks. */
1564 if (IS_I945G(dev) || IS_I945GM(dev))
1565 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
1566 else if (IS_I915GM(dev))
1567 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_SELF_EN));
1568
1569 /* Calc sr entries for one plane configs */
1570 if (HAS_FW_BLC(dev) && enabled) {
1571 /* self-refresh has much higher latency */
1572 static const int sr_latency_ns = 6000;
1573 const struct drm_display_mode *adjusted_mode =
1574 &to_intel_crtc(enabled)->config.adjusted_mode;
1575 int clock = adjusted_mode->crtc_clock;
1576 int htotal = adjusted_mode->crtc_htotal;
1577 int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
1578 int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
1579 unsigned long line_time_us;
1580 int entries;
1581
1582 line_time_us = max(htotal * 1000 / clock, 1);
1583
1584 /* Use ns/us then divide to preserve precision */
1585 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1586 pixel_size * hdisplay;
1587 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1588 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1589 srwm = wm_info->fifo_size - entries;
1590 if (srwm < 0)
1591 srwm = 1;
1592
1593 if (IS_I945G(dev) || IS_I945GM(dev))
1594 I915_WRITE(FW_BLC_SELF,
1595 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1596 else if (IS_I915GM(dev))
1597 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1598 }
1599
1600 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1601 planea_wm, planeb_wm, cwm, srwm);
1602
1603 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1604 fwater_hi = (cwm & 0x1f);
1605
1606 /* Set request length to 8 cachelines per fetch */
1607 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1608 fwater_hi = fwater_hi | (1 << 8);
1609
1610 I915_WRITE(FW_BLC, fwater_lo);
1611 I915_WRITE(FW_BLC2, fwater_hi);
1612
1613 if (HAS_FW_BLC(dev)) {
1614 if (enabled) {
1615 if (IS_I945G(dev) || IS_I945GM(dev))
1616 I915_WRITE(FW_BLC_SELF,
1617 FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
1618 else if (IS_I915GM(dev))
1619 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_SELF_EN));
1620 DRM_DEBUG_KMS("memory self refresh enabled\n");
1621 } else
1622 DRM_DEBUG_KMS("memory self refresh disabled\n");
1623 }
1624}
1625
1626static void i845_update_wm(struct drm_crtc *unused_crtc)
1627{
1628 struct drm_device *dev = unused_crtc->dev;
1629 struct drm_i915_private *dev_priv = dev->dev_private;
1630 struct drm_crtc *crtc;
1631 const struct drm_display_mode *adjusted_mode;
1632 uint32_t fwater_lo;
1633 int planea_wm;
1634
1635 crtc = single_enabled_crtc(dev);
1636 if (crtc == NULL)
1637 return;
1638
1639 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1640 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1641 &i845_wm_info,
1642 dev_priv->display.get_fifo_size(dev, 0),
1643 4, latency_ns);
1644 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1645 fwater_lo |= (3<<8) | planea_wm;
1646
1647 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1648
1649 I915_WRITE(FW_BLC, fwater_lo);
1650}
1651
1652static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1653 struct drm_crtc *crtc)
1654{
1655 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1656 uint32_t pixel_rate;
1657
1658 pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
1659
1660 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1661 * adjust the pixel_rate here. */
1662
1663 if (intel_crtc->config.pch_pfit.enabled) {
1664 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
1665 uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
1666
1667 pipe_w = intel_crtc->config.pipe_src_w;
1668 pipe_h = intel_crtc->config.pipe_src_h;
1669 pfit_w = (pfit_size >> 16) & 0xFFFF;
1670 pfit_h = pfit_size & 0xFFFF;
1671 if (pipe_w < pfit_w)
1672 pipe_w = pfit_w;
1673 if (pipe_h < pfit_h)
1674 pipe_h = pfit_h;
1675
1676 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1677 pfit_w * pfit_h);
1678 }
1679
1680 return pixel_rate;
1681}
1682
1683/* latency must be in 0.1us units. */
1684static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
1685 uint32_t latency)
1686{
1687 uint64_t ret;
1688
1689 if (WARN(latency == 0, "Latency value missing\n"))
1690 return UINT_MAX;
1691
1692 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1693 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1694
1695 return ret;
1696}
1697
1698/* latency must be in 0.1us units. */
1699static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1700 uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1701 uint32_t latency)
1702{
1703 uint32_t ret;
1704
1705 if (WARN(latency == 0, "Latency value missing\n"))
1706 return UINT_MAX;
1707
1708 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1709 ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1710 ret = DIV_ROUND_UP(ret, 64) + 2;
1711 return ret;
1712}
1713
1714static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1715 uint8_t bytes_per_pixel)
1716{
1717 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1718}
1719
1720struct ilk_pipe_wm_parameters {
1721 bool active;
1722 uint32_t pipe_htotal;
1723 uint32_t pixel_rate;
1724 struct intel_plane_wm_parameters pri;
1725 struct intel_plane_wm_parameters spr;
1726 struct intel_plane_wm_parameters cur;
1727};
1728
1729struct ilk_wm_maximums {
1730 uint16_t pri;
1731 uint16_t spr;
1732 uint16_t cur;
1733 uint16_t fbc;
1734};
1735
1736/* used in computing the new watermarks state */
1737struct intel_wm_config {
1738 unsigned int num_pipes_active;
1739 bool sprites_enabled;
1740 bool sprites_scaled;
1741};
1742
1743/*
1744 * For both WM_PIPE and WM_LP.
1745 * mem_value must be in 0.1us units.
1746 */
1747static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
1748 uint32_t mem_value,
1749 bool is_lp)
1750{
1751 uint32_t method1, method2;
1752
1753 if (!params->active || !params->pri.enabled)
1754 return 0;
1755
1756 method1 = ilk_wm_method1(params->pixel_rate,
1757 params->pri.bytes_per_pixel,
1758 mem_value);
1759
1760 if (!is_lp)
1761 return method1;
1762
1763 method2 = ilk_wm_method2(params->pixel_rate,
1764 params->pipe_htotal,
1765 params->pri.horiz_pixels,
1766 params->pri.bytes_per_pixel,
1767 mem_value);
1768
1769 return min(method1, method2);
1770}
1771
1772/*
1773 * For both WM_PIPE and WM_LP.
1774 * mem_value must be in 0.1us units.
1775 */
1776static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
1777 uint32_t mem_value)
1778{
1779 uint32_t method1, method2;
1780
1781 if (!params->active || !params->spr.enabled)
1782 return 0;
1783
1784 method1 = ilk_wm_method1(params->pixel_rate,
1785 params->spr.bytes_per_pixel,
1786 mem_value);
1787 method2 = ilk_wm_method2(params->pixel_rate,
1788 params->pipe_htotal,
1789 params->spr.horiz_pixels,
1790 params->spr.bytes_per_pixel,
1791 mem_value);
1792 return min(method1, method2);
1793}
1794
1795/*
1796 * For both WM_PIPE and WM_LP.
1797 * mem_value must be in 0.1us units.
1798 */
1799static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
1800 uint32_t mem_value)
1801{
1802 if (!params->active || !params->cur.enabled)
1803 return 0;
1804
1805 return ilk_wm_method2(params->pixel_rate,
1806 params->pipe_htotal,
1807 params->cur.horiz_pixels,
1808 params->cur.bytes_per_pixel,
1809 mem_value);
1810}
1811
1812/* Only for WM_LP. */
1813static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1814 uint32_t pri_val)
1815{
1816 if (!params->active || !params->pri.enabled)
1817 return 0;
1818
1819 return ilk_wm_fbc(pri_val,
1820 params->pri.horiz_pixels,
1821 params->pri.bytes_per_pixel);
1822}
1823
1824static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1825{
1826 if (INTEL_INFO(dev)->gen >= 8)
1827 return 3072;
1828 else if (INTEL_INFO(dev)->gen >= 7)
1829 return 768;
1830 else
1831 return 512;
1832}
1833
1834/* Calculate the maximum primary/sprite plane watermark */
1835static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1836 int level,
1837 const struct intel_wm_config *config,
1838 enum intel_ddb_partitioning ddb_partitioning,
1839 bool is_sprite)
1840{
1841 unsigned int fifo_size = ilk_display_fifo_size(dev);
1842 unsigned int max;
1843
1844 /* if sprites aren't enabled, sprites get nothing */
1845 if (is_sprite && !config->sprites_enabled)
1846 return 0;
1847
1848 /* HSW allows LP1+ watermarks even with multiple pipes */
1849 if (level == 0 || config->num_pipes_active > 1) {
1850 fifo_size /= INTEL_INFO(dev)->num_pipes;
1851
1852 /*
1853 * For some reason the non self refresh
1854 * FIFO size is only half of the self
1855 * refresh FIFO size on ILK/SNB.
1856 */
1857 if (INTEL_INFO(dev)->gen <= 6)
1858 fifo_size /= 2;
1859 }
1860
1861 if (config->sprites_enabled) {
1862 /* level 0 is always calculated with 1:1 split */
1863 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1864 if (is_sprite)
1865 fifo_size *= 5;
1866 fifo_size /= 6;
1867 } else {
1868 fifo_size /= 2;
1869 }
1870 }
1871
1872 /* clamp to max that the registers can hold */
1873 if (INTEL_INFO(dev)->gen >= 8)
1874 max = level == 0 ? 255 : 2047;
1875 else if (INTEL_INFO(dev)->gen >= 7)
1876 /* IVB/HSW primary/sprite plane watermarks */
1877 max = level == 0 ? 127 : 1023;
1878 else if (!is_sprite)
1879 /* ILK/SNB primary plane watermarks */
1880 max = level == 0 ? 127 : 511;
1881 else
1882 /* ILK/SNB sprite plane watermarks */
1883 max = level == 0 ? 63 : 255;
1884
1885 return min(fifo_size, max);
1886}
1887
1888/* Calculate the maximum cursor plane watermark */
1889static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1890 int level,
1891 const struct intel_wm_config *config)
1892{
1893 /* HSW LP1+ watermarks w/ multiple pipes */
1894 if (level > 0 && config->num_pipes_active > 1)
1895 return 64;
1896
1897 /* otherwise just report max that registers can hold */
1898 if (INTEL_INFO(dev)->gen >= 7)
1899 return level == 0 ? 63 : 255;
1900 else
1901 return level == 0 ? 31 : 63;
1902}
1903
1904/* Calculate the maximum FBC watermark */
1905static unsigned int ilk_fbc_wm_max(const struct drm_device *dev)
1906{
1907 /* max that registers can hold */
1908 if (INTEL_INFO(dev)->gen >= 8)
1909 return 31;
1910 else
1911 return 15;
1912}
1913
1914static void ilk_compute_wm_maximums(const struct drm_device *dev,
1915 int level,
1916 const struct intel_wm_config *config,
1917 enum intel_ddb_partitioning ddb_partitioning,
1918 struct ilk_wm_maximums *max)
1919{
1920 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1921 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1922 max->cur = ilk_cursor_wm_max(dev, level, config);
1923 max->fbc = ilk_fbc_wm_max(dev);
1924}
1925
1926static bool ilk_validate_wm_level(int level,
1927 const struct ilk_wm_maximums *max,
1928 struct intel_wm_level *result)
1929{
1930 bool ret;
1931
1932 /* already determined to be invalid? */
1933 if (!result->enable)
1934 return false;
1935
1936 result->enable = result->pri_val <= max->pri &&
1937 result->spr_val <= max->spr &&
1938 result->cur_val <= max->cur;
1939
1940 ret = result->enable;
1941
1942 /*
1943 * HACK until we can pre-compute everything,
1944 * and thus fail gracefully if LP0 watermarks
1945 * are exceeded...
1946 */
1947 if (level == 0 && !result->enable) {
1948 if (result->pri_val > max->pri)
1949 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1950 level, result->pri_val, max->pri);
1951 if (result->spr_val > max->spr)
1952 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1953 level, result->spr_val, max->spr);
1954 if (result->cur_val > max->cur)
1955 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1956 level, result->cur_val, max->cur);
1957
1958 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1959 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1960 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1961 result->enable = true;
1962 }
1963
1964 return ret;
1965}
1966
1967static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
1968 int level,
1969 const struct ilk_pipe_wm_parameters *p,
1970 struct intel_wm_level *result)
1971{
1972 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
1973 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
1974 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
1975
1976 /* WM1+ latency values stored in 0.5us units */
1977 if (level > 0) {
1978 pri_latency *= 5;
1979 spr_latency *= 5;
1980 cur_latency *= 5;
1981 }
1982
1983 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
1984 result->spr_val = ilk_compute_spr_wm(p, spr_latency);
1985 result->cur_val = ilk_compute_cur_wm(p, cur_latency);
1986 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
1987 result->enable = true;
1988}
1989
1990static uint32_t
1991hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
1992{
1993 struct drm_i915_private *dev_priv = dev->dev_private;
1994 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1995 struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
1996 u32 linetime, ips_linetime;
1997
1998 if (!intel_crtc_active(crtc))
1999 return 0;
2000
2001 /* The WM are computed with base on how long it takes to fill a single
2002 * row at the given clock rate, multiplied by 8.
2003 * */
2004 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
2005 mode->crtc_clock);
2006 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
2007 intel_ddi_get_cdclk_freq(dev_priv));
2008
2009 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2010 PIPE_WM_LINETIME_TIME(linetime);
2011}
2012
2013static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
2014{
2015 struct drm_i915_private *dev_priv = dev->dev_private;
2016
2017 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2018 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2019
2020 wm[0] = (sskpd >> 56) & 0xFF;
2021 if (wm[0] == 0)
2022 wm[0] = sskpd & 0xF;
2023 wm[1] = (sskpd >> 4) & 0xFF;
2024 wm[2] = (sskpd >> 12) & 0xFF;
2025 wm[3] = (sskpd >> 20) & 0x1FF;
2026 wm[4] = (sskpd >> 32) & 0x1FF;
2027 } else if (INTEL_INFO(dev)->gen >= 6) {
2028 uint32_t sskpd = I915_READ(MCH_SSKPD);
2029
2030 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2031 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2032 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2033 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2034 } else if (INTEL_INFO(dev)->gen >= 5) {
2035 uint32_t mltr = I915_READ(MLTR_ILK);
2036
2037 /* ILK primary LP0 latency is 700 ns */
2038 wm[0] = 7;
2039 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2040 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2041 }
2042}
2043
2044static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2045{
2046 /* ILK sprite LP0 latency is 1300 ns */
2047 if (INTEL_INFO(dev)->gen == 5)
2048 wm[0] = 13;
2049}
2050
2051static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2052{
2053 /* ILK cursor LP0 latency is 1300 ns */
2054 if (INTEL_INFO(dev)->gen == 5)
2055 wm[0] = 13;
2056
2057 /* WaDoubleCursorLP3Latency:ivb */
2058 if (IS_IVYBRIDGE(dev))
2059 wm[3] *= 2;
2060}
2061
2062static int ilk_wm_max_level(const struct drm_device *dev)
2063{
2064 /* how many WM levels are we expecting */
2065 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2066 return 4;
2067 else if (INTEL_INFO(dev)->gen >= 6)
2068 return 3;
2069 else
2070 return 2;
2071}
2072
2073static void intel_print_wm_latency(struct drm_device *dev,
2074 const char *name,
2075 const uint16_t wm[5])
2076{
2077 int level, max_level = ilk_wm_max_level(dev);
2078
2079 for (level = 0; level <= max_level; level++) {
2080 unsigned int latency = wm[level];
2081
2082 if (latency == 0) {
2083 DRM_ERROR("%s WM%d latency not provided\n",
2084 name, level);
2085 continue;
2086 }
2087
2088 /* WM1+ latency values in 0.5us units */
2089 if (level > 0)
2090 latency *= 5;
2091
2092 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2093 name, level, wm[level],
2094 latency / 10, latency % 10);
2095 }
2096}
2097
2098static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2099 uint16_t wm[5], uint16_t min)
2100{
2101 int level, max_level = ilk_wm_max_level(dev_priv->dev);
2102
2103 if (wm[0] >= min)
2104 return false;
2105
2106 wm[0] = max(wm[0], min);
2107 for (level = 1; level <= max_level; level++)
2108 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2109
2110 return true;
2111}
2112
2113static void snb_wm_latency_quirk(struct drm_device *dev)
2114{
2115 struct drm_i915_private *dev_priv = dev->dev_private;
2116 bool changed;
2117
2118 /*
2119 * The BIOS provided WM memory latency values are often
2120 * inadequate for high resolution displays. Adjust them.
2121 */
2122 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2123 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2124 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2125
2126 if (!changed)
2127 return;
2128
2129 DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2130 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2131 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2132 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2133}
2134
2135static void ilk_setup_wm_latency(struct drm_device *dev)
2136{
2137 struct drm_i915_private *dev_priv = dev->dev_private;
2138
2139 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2140
2141 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2142 sizeof(dev_priv->wm.pri_latency));
2143 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2144 sizeof(dev_priv->wm.pri_latency));
2145
2146 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2147 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
2148
2149 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2150 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2151 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2152
2153 if (IS_GEN6(dev))
2154 snb_wm_latency_quirk(dev);
2155}
2156
2157static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2158 struct ilk_pipe_wm_parameters *p,
2159 struct intel_wm_config *config)
2160{
2161 struct drm_device *dev = crtc->dev;
2162 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2163 enum pipe pipe = intel_crtc->pipe;
2164 struct drm_plane *plane;
2165
2166 p->active = intel_crtc_active(crtc);
2167 if (p->active) {
2168 p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
2169 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
2170 p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
2171 p->cur.bytes_per_pixel = 4;
2172 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
2173 p->cur.horiz_pixels = intel_crtc->cursor_width;
2174 /* TODO: for now, assume primary and cursor planes are always enabled. */
2175 p->pri.enabled = true;
2176 p->cur.enabled = true;
2177 }
2178
2179 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2180 config->num_pipes_active += intel_crtc_active(crtc);
2181
2182 drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
2183 struct intel_plane *intel_plane = to_intel_plane(plane);
2184
2185 if (intel_plane->pipe == pipe)
2186 p->spr = intel_plane->wm;
2187
2188 config->sprites_enabled |= intel_plane->wm.enabled;
2189 config->sprites_scaled |= intel_plane->wm.scaled;
2190 }
2191}
2192
2193/* Compute new watermarks for the pipe */
2194static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
2195 const struct ilk_pipe_wm_parameters *params,
2196 struct intel_pipe_wm *pipe_wm)
2197{
2198 struct drm_device *dev = crtc->dev;
2199 const struct drm_i915_private *dev_priv = dev->dev_private;
2200 int level, max_level = ilk_wm_max_level(dev);
2201 /* LP0 watermark maximums depend on this pipe alone */
2202 struct intel_wm_config config = {
2203 .num_pipes_active = 1,
2204 .sprites_enabled = params->spr.enabled,
2205 .sprites_scaled = params->spr.scaled,
2206 };
2207 struct ilk_wm_maximums max;
2208
2209 /* LP0 watermarks always use 1/2 DDB partitioning */
2210 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2211
2212 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2213 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2214 max_level = 1;
2215
2216 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2217 if (params->spr.scaled)
2218 max_level = 0;
2219
2220 for (level = 0; level <= max_level; level++)
2221 ilk_compute_wm_level(dev_priv, level, params,
2222 &pipe_wm->wm[level]);
2223
2224 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2225 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
2226
2227 /* At least LP0 must be valid */
2228 return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]);
2229}
2230
2231/*
2232 * Merge the watermarks from all active pipes for a specific level.
2233 */
2234static void ilk_merge_wm_level(struct drm_device *dev,
2235 int level,
2236 struct intel_wm_level *ret_wm)
2237{
2238 const struct intel_crtc *intel_crtc;
2239
2240 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2241 const struct intel_wm_level *wm =
2242 &intel_crtc->wm.active.wm[level];
2243
2244 if (!wm->enable)
2245 return;
2246
2247 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2248 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2249 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2250 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2251 }
2252
2253 ret_wm->enable = true;
2254}
2255
2256/*
2257 * Merge all low power watermarks for all active pipes.
2258 */
2259static void ilk_wm_merge(struct drm_device *dev,
2260 const struct intel_wm_config *config,
2261 const struct ilk_wm_maximums *max,
2262 struct intel_pipe_wm *merged)
2263{
2264 int level, max_level = ilk_wm_max_level(dev);
2265
2266 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2267 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2268 config->num_pipes_active > 1)
2269 return;
2270
2271 /* ILK: FBC WM must be disabled always */
2272 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2273
2274 /* merge each WM1+ level */
2275 for (level = 1; level <= max_level; level++) {
2276 struct intel_wm_level *wm = &merged->wm[level];
2277
2278 ilk_merge_wm_level(dev, level, wm);
2279
2280 if (!ilk_validate_wm_level(level, max, wm))
2281 break;
2282
2283 /*
2284 * The spec says it is preferred to disable
2285 * FBC WMs instead of disabling a WM level.
2286 */
2287 if (wm->fbc_val > max->fbc) {
2288 merged->fbc_wm_enabled = false;
2289 wm->fbc_val = 0;
2290 }
2291 }
2292
2293 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2294 /*
2295 * FIXME this is racy. FBC might get enabled later.
2296 * What we should check here is whether FBC can be
2297 * enabled sometime later.
2298 */
2299 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2300 for (level = 2; level <= max_level; level++) {
2301 struct intel_wm_level *wm = &merged->wm[level];
2302
2303 wm->enable = false;
2304 }
2305 }
2306}
2307
2308static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2309{
2310 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2311 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2312}
2313
2314/* The value we need to program into the WM_LPx latency field */
2315static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2316{
2317 struct drm_i915_private *dev_priv = dev->dev_private;
2318
2319 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2320 return 2 * level;
2321 else
2322 return dev_priv->wm.pri_latency[level];
2323}
2324
2325static void ilk_compute_wm_results(struct drm_device *dev,
2326 const struct intel_pipe_wm *merged,
2327 enum intel_ddb_partitioning partitioning,
2328 struct ilk_wm_values *results)
2329{
2330 struct intel_crtc *intel_crtc;
2331 int level, wm_lp;
2332
2333 results->enable_fbc_wm = merged->fbc_wm_enabled;
2334 results->partitioning = partitioning;
2335
2336 /* LP1+ register values */
2337 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2338 const struct intel_wm_level *r;
2339
2340 level = ilk_wm_lp_to_level(wm_lp, merged);
2341
2342 r = &merged->wm[level];
2343 if (!r->enable)
2344 break;
2345
2346 results->wm_lp[wm_lp - 1] = WM3_LP_EN |
2347 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2348 (r->pri_val << WM1_LP_SR_SHIFT) |
2349 r->cur_val;
2350
2351 if (INTEL_INFO(dev)->gen >= 8)
2352 results->wm_lp[wm_lp - 1] |=
2353 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2354 else
2355 results->wm_lp[wm_lp - 1] |=
2356 r->fbc_val << WM1_LP_FBC_SHIFT;
2357
2358 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2359 WARN_ON(wm_lp != 1);
2360 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2361 } else
2362 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2363 }
2364
2365 /* LP0 register values */
2366 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2367 enum pipe pipe = intel_crtc->pipe;
2368 const struct intel_wm_level *r =
2369 &intel_crtc->wm.active.wm[0];
2370
2371 if (WARN_ON(!r->enable))
2372 continue;
2373
2374 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2375
2376 results->wm_pipe[pipe] =
2377 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2378 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2379 r->cur_val;
2380 }
2381}
2382
2383/* Find the result with the highest level enabled. Check for enable_fbc_wm in
2384 * case both are at the same level. Prefer r1 in case they're the same. */
2385static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2386 struct intel_pipe_wm *r1,
2387 struct intel_pipe_wm *r2)
2388{
2389 int level, max_level = ilk_wm_max_level(dev);
2390 int level1 = 0, level2 = 0;
2391
2392 for (level = 1; level <= max_level; level++) {
2393 if (r1->wm[level].enable)
2394 level1 = level;
2395 if (r2->wm[level].enable)
2396 level2 = level;
2397 }
2398
2399 if (level1 == level2) {
2400 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2401 return r2;
2402 else
2403 return r1;
2404 } else if (level1 > level2) {
2405 return r1;
2406 } else {
2407 return r2;
2408 }
2409}
2410
2411/* dirty bits used to track which watermarks need changes */
2412#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2413#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2414#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2415#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2416#define WM_DIRTY_FBC (1 << 24)
2417#define WM_DIRTY_DDB (1 << 25)
2418
2419static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
2420 const struct ilk_wm_values *old,
2421 const struct ilk_wm_values *new)
2422{
2423 unsigned int dirty = 0;
2424 enum pipe pipe;
2425 int wm_lp;
2426
2427 for_each_pipe(pipe) {
2428 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2429 dirty |= WM_DIRTY_LINETIME(pipe);
2430 /* Must disable LP1+ watermarks too */
2431 dirty |= WM_DIRTY_LP_ALL;
2432 }
2433
2434 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2435 dirty |= WM_DIRTY_PIPE(pipe);
2436 /* Must disable LP1+ watermarks too */
2437 dirty |= WM_DIRTY_LP_ALL;
2438 }
2439 }
2440
2441 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2442 dirty |= WM_DIRTY_FBC;
2443 /* Must disable LP1+ watermarks too */
2444 dirty |= WM_DIRTY_LP_ALL;
2445 }
2446
2447 if (old->partitioning != new->partitioning) {
2448 dirty |= WM_DIRTY_DDB;
2449 /* Must disable LP1+ watermarks too */
2450 dirty |= WM_DIRTY_LP_ALL;
2451 }
2452
2453 /* LP1+ watermarks already deemed dirty, no need to continue */
2454 if (dirty & WM_DIRTY_LP_ALL)
2455 return dirty;
2456
2457 /* Find the lowest numbered LP1+ watermark in need of an update... */
2458 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2459 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2460 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2461 break;
2462 }
2463
2464 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2465 for (; wm_lp <= 3; wm_lp++)
2466 dirty |= WM_DIRTY_LP(wm_lp);
2467
2468 return dirty;
2469}
2470
2471static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2472 unsigned int dirty)
2473{
2474 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2475 bool changed = false;
2476
2477 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2478 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2479 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2480 changed = true;
2481 }
2482 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2483 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2484 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2485 changed = true;
2486 }
2487 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2488 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2489 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2490 changed = true;
2491 }
2492
2493 /*
2494 * Don't touch WM1S_LP_EN here.
2495 * Doing so could cause underruns.
2496 */
2497
2498 return changed;
2499}
2500
2501/*
2502 * The spec says we shouldn't write when we don't need, because every write
2503 * causes WMs to be re-evaluated, expending some power.
2504 */
2505static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2506 struct ilk_wm_values *results)
2507{
2508 struct drm_device *dev = dev_priv->dev;
2509 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2510 unsigned int dirty;
2511 uint32_t val;
2512
2513 dirty = ilk_compute_wm_dirty(dev, previous, results);
2514 if (!dirty)
2515 return;
2516
2517 _ilk_disable_lp_wm(dev_priv, dirty);
2518
2519 if (dirty & WM_DIRTY_PIPE(PIPE_A))
2520 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2521 if (dirty & WM_DIRTY_PIPE(PIPE_B))
2522 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2523 if (dirty & WM_DIRTY_PIPE(PIPE_C))
2524 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2525
2526 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2527 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2528 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2529 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2530 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2531 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2532
2533 if (dirty & WM_DIRTY_DDB) {
2534 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2535 val = I915_READ(WM_MISC);
2536 if (results->partitioning == INTEL_DDB_PART_1_2)
2537 val &= ~WM_MISC_DATA_PARTITION_5_6;
2538 else
2539 val |= WM_MISC_DATA_PARTITION_5_6;
2540 I915_WRITE(WM_MISC, val);
2541 } else {
2542 val = I915_READ(DISP_ARB_CTL2);
2543 if (results->partitioning == INTEL_DDB_PART_1_2)
2544 val &= ~DISP_DATA_PARTITION_5_6;
2545 else
2546 val |= DISP_DATA_PARTITION_5_6;
2547 I915_WRITE(DISP_ARB_CTL2, val);
2548 }
2549 }
2550
2551 if (dirty & WM_DIRTY_FBC) {
2552 val = I915_READ(DISP_ARB_CTL);
2553 if (results->enable_fbc_wm)
2554 val &= ~DISP_FBC_WM_DIS;
2555 else
2556 val |= DISP_FBC_WM_DIS;
2557 I915_WRITE(DISP_ARB_CTL, val);
2558 }
2559
2560 if (dirty & WM_DIRTY_LP(1) &&
2561 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2562 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2563
2564 if (INTEL_INFO(dev)->gen >= 7) {
2565 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2566 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2567 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2568 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2569 }
2570
2571 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
2572 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2573 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
2574 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2575 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
2576 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2577
2578 dev_priv->wm.hw = *results;
2579}
2580
2581static bool ilk_disable_lp_wm(struct drm_device *dev)
2582{
2583 struct drm_i915_private *dev_priv = dev->dev_private;
2584
2585 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2586}
2587
2588static void ilk_update_wm(struct drm_crtc *crtc)
2589{
2590 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2591 struct drm_device *dev = crtc->dev;
2592 struct drm_i915_private *dev_priv = dev->dev_private;
2593 struct ilk_wm_maximums max;
2594 struct ilk_pipe_wm_parameters params = {};
2595 struct ilk_wm_values results = {};
2596 enum intel_ddb_partitioning partitioning;
2597 struct intel_pipe_wm pipe_wm = {};
2598 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
2599 struct intel_wm_config config = {};
2600
2601 ilk_compute_wm_parameters(crtc, ¶ms, &config);
2602
2603 intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
2604
2605 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
2606 return;
2607
2608 intel_crtc->wm.active = pipe_wm;
2609
2610 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
2611 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
2612
2613 /* 5/6 split only in single pipe config on IVB+ */
2614 if (INTEL_INFO(dev)->gen >= 7 &&
2615 config.num_pipes_active == 1 && config.sprites_enabled) {
2616 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
2617 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
2618
2619 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
2620 } else {
2621 best_lp_wm = &lp_wm_1_2;
2622 }
2623
2624 partitioning = (best_lp_wm == &lp_wm_1_2) ?
2625 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
2626
2627 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
2628
2629 ilk_write_wm_values(dev_priv, &results);
2630}
2631
2632static void ilk_update_sprite_wm(struct drm_plane *plane,
2633 struct drm_crtc *crtc,
2634 uint32_t sprite_width, int pixel_size,
2635 bool enabled, bool scaled)
2636{
2637 struct drm_device *dev = plane->dev;
2638 struct intel_plane *intel_plane = to_intel_plane(plane);
2639
2640 intel_plane->wm.enabled = enabled;
2641 intel_plane->wm.scaled = scaled;
2642 intel_plane->wm.horiz_pixels = sprite_width;
2643 intel_plane->wm.bytes_per_pixel = pixel_size;
2644
2645 /*
2646 * IVB workaround: must disable low power watermarks for at least
2647 * one frame before enabling scaling. LP watermarks can be re-enabled
2648 * when scaling is disabled.
2649 *
2650 * WaCxSRDisabledForSpriteScaling:ivb
2651 */
2652 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
2653 intel_wait_for_vblank(dev, intel_plane->pipe);
2654
2655 ilk_update_wm(crtc);
2656}
2657
2658static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
2659{
2660 struct drm_device *dev = crtc->dev;
2661 struct drm_i915_private *dev_priv = dev->dev_private;
2662 struct ilk_wm_values *hw = &dev_priv->wm.hw;
2663 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2664 struct intel_pipe_wm *active = &intel_crtc->wm.active;
2665 enum pipe pipe = intel_crtc->pipe;
2666 static const unsigned int wm0_pipe_reg[] = {
2667 [PIPE_A] = WM0_PIPEA_ILK,
2668 [PIPE_B] = WM0_PIPEB_ILK,
2669 [PIPE_C] = WM0_PIPEC_IVB,
2670 };
2671
2672 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
2673 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2674 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
2675
2676 if (intel_crtc_active(crtc)) {
2677 u32 tmp = hw->wm_pipe[pipe];
2678
2679 /*
2680 * For active pipes LP0 watermark is marked as
2681 * enabled, and LP1+ watermaks as disabled since
2682 * we can't really reverse compute them in case
2683 * multiple pipes are active.
2684 */
2685 active->wm[0].enable = true;
2686 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
2687 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
2688 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
2689 active->linetime = hw->wm_linetime[pipe];
2690 } else {
2691 int level, max_level = ilk_wm_max_level(dev);
2692
2693 /*
2694 * For inactive pipes, all watermark levels
2695 * should be marked as enabled but zeroed,
2696 * which is what we'd compute them to.
2697 */
2698 for (level = 0; level <= max_level; level++)
2699 active->wm[level].enable = true;
2700 }
2701}
2702
2703void ilk_wm_get_hw_state(struct drm_device *dev)
2704{
2705 struct drm_i915_private *dev_priv = dev->dev_private;
2706 struct ilk_wm_values *hw = &dev_priv->wm.hw;
2707 struct drm_crtc *crtc;
2708
2709 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2710 ilk_pipe_wm_get_hw_state(crtc);
2711
2712 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
2713 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
2714 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
2715
2716 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
2717 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
2718 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
2719
2720 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2721 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
2722 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
2723 else if (IS_IVYBRIDGE(dev))
2724 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
2725 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
2726
2727 hw->enable_fbc_wm =
2728 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
2729}
2730
2731/**
2732 * intel_update_watermarks - update FIFO watermark values based on current modes
2733 *
2734 * Calculate watermark values for the various WM regs based on current mode
2735 * and plane configuration.
2736 *
2737 * There are several cases to deal with here:
2738 * - normal (i.e. non-self-refresh)
2739 * - self-refresh (SR) mode
2740 * - lines are large relative to FIFO size (buffer can hold up to 2)
2741 * - lines are small relative to FIFO size (buffer can hold more than 2
2742 * lines), so need to account for TLB latency
2743 *
2744 * The normal calculation is:
2745 * watermark = dotclock * bytes per pixel * latency
2746 * where latency is platform & configuration dependent (we assume pessimal
2747 * values here).
2748 *
2749 * The SR calculation is:
2750 * watermark = (trunc(latency/line time)+1) * surface width *
2751 * bytes per pixel
2752 * where
2753 * line time = htotal / dotclock
2754 * surface width = hdisplay for normal plane and 64 for cursor
2755 * and latency is assumed to be high, as above.
2756 *
2757 * The final value programmed to the register should always be rounded up,
2758 * and include an extra 2 entries to account for clock crossings.
2759 *
2760 * We don't use the sprite, so we can ignore that. And on Crestline we have
2761 * to set the non-SR watermarks to 8.
2762 */
2763void intel_update_watermarks(struct drm_crtc *crtc)
2764{
2765 struct drm_i915_private *dev_priv = crtc->dev->dev_private;
2766
2767 if (dev_priv->display.update_wm)
2768 dev_priv->display.update_wm(crtc);
2769}
2770
2771void intel_update_sprite_watermarks(struct drm_plane *plane,
2772 struct drm_crtc *crtc,
2773 uint32_t sprite_width, int pixel_size,
2774 bool enabled, bool scaled)
2775{
2776 struct drm_i915_private *dev_priv = plane->dev->dev_private;
2777
2778 if (dev_priv->display.update_sprite_wm)
2779 dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
2780 pixel_size, enabled, scaled);
2781}
2782
2783static struct drm_i915_gem_object *
2784intel_alloc_context_page(struct drm_device *dev)
2785{
2786 struct drm_i915_gem_object *ctx;
2787 int ret;
2788
2789 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2790
2791 ctx = i915_gem_alloc_object(dev, 4096);
2792 if (!ctx) {
2793 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
2794 return NULL;
2795 }
2796
2797 ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
2798 if (ret) {
2799 DRM_ERROR("failed to pin power context: %d\n", ret);
2800 goto err_unref;
2801 }
2802
2803 ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
2804 if (ret) {
2805 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
2806 goto err_unpin;
2807 }
2808
2809 return ctx;
2810
2811err_unpin:
2812 i915_gem_object_ggtt_unpin(ctx);
2813err_unref:
2814 drm_gem_object_unreference(&ctx->base);
2815 return NULL;
2816}
2817
2818/**
2819 * Lock protecting IPS related data structures
2820 */
2821DEFINE_SPINLOCK(mchdev_lock);
2822
2823/* Global for IPS driver to get at the current i915 device. Protected by
2824 * mchdev_lock. */
2825static struct drm_i915_private *i915_mch_dev;
2826
2827bool ironlake_set_drps(struct drm_device *dev, u8 val)
2828{
2829 struct drm_i915_private *dev_priv = dev->dev_private;
2830 u16 rgvswctl;
2831
2832 assert_spin_locked(&mchdev_lock);
2833
2834 rgvswctl = I915_READ16(MEMSWCTL);
2835 if (rgvswctl & MEMCTL_CMD_STS) {
2836 DRM_DEBUG("gpu busy, RCS change rejected\n");
2837 return false; /* still busy with another command */
2838 }
2839
2840 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
2841 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
2842 I915_WRITE16(MEMSWCTL, rgvswctl);
2843 POSTING_READ16(MEMSWCTL);
2844
2845 rgvswctl |= MEMCTL_CMD_STS;
2846 I915_WRITE16(MEMSWCTL, rgvswctl);
2847
2848 return true;
2849}
2850
2851static void ironlake_enable_drps(struct drm_device *dev)
2852{
2853 struct drm_i915_private *dev_priv = dev->dev_private;
2854 u32 rgvmodectl = I915_READ(MEMMODECTL);
2855 u8 fmax, fmin, fstart, vstart;
2856
2857 spin_lock_irq(&mchdev_lock);
2858
2859 /* Enable temp reporting */
2860 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
2861 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
2862
2863 /* 100ms RC evaluation intervals */
2864 I915_WRITE(RCUPEI, 100000);
2865 I915_WRITE(RCDNEI, 100000);
2866
2867 /* Set max/min thresholds to 90ms and 80ms respectively */
2868 I915_WRITE(RCBMAXAVG, 90000);
2869 I915_WRITE(RCBMINAVG, 80000);
2870
2871 I915_WRITE(MEMIHYST, 1);
2872
2873 /* Set up min, max, and cur for interrupt handling */
2874 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
2875 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
2876 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
2877 MEMMODE_FSTART_SHIFT;
2878
2879 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
2880 PXVFREQ_PX_SHIFT;
2881
2882 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
2883 dev_priv->ips.fstart = fstart;
2884
2885 dev_priv->ips.max_delay = fstart;
2886 dev_priv->ips.min_delay = fmin;
2887 dev_priv->ips.cur_delay = fstart;
2888
2889 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
2890 fmax, fmin, fstart);
2891
2892 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
2893
2894 /*
2895 * Interrupts will be enabled in ironlake_irq_postinstall
2896 */
2897
2898 I915_WRITE(VIDSTART, vstart);
2899 POSTING_READ(VIDSTART);
2900
2901 rgvmodectl |= MEMMODE_SWMODE_EN;
2902 I915_WRITE(MEMMODECTL, rgvmodectl);
2903
2904 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
2905 DRM_ERROR("stuck trying to change perf mode\n");
2906 mdelay(1);
2907
2908 ironlake_set_drps(dev, fstart);
2909
2910 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
2911 I915_READ(0x112e0);
2912 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
2913 dev_priv->ips.last_count2 = I915_READ(0x112f4);
2914 getrawmonotonic(&dev_priv->ips.last_time2);
2915
2916 spin_unlock_irq(&mchdev_lock);
2917}
2918
2919static void ironlake_disable_drps(struct drm_device *dev)
2920{
2921 struct drm_i915_private *dev_priv = dev->dev_private;
2922 u16 rgvswctl;
2923
2924 spin_lock_irq(&mchdev_lock);
2925
2926 rgvswctl = I915_READ16(MEMSWCTL);
2927
2928 /* Ack interrupts, disable EFC interrupt */
2929 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
2930 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
2931 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
2932 I915_WRITE(DEIIR, DE_PCU_EVENT);
2933 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
2934
2935 /* Go back to the starting frequency */
2936 ironlake_set_drps(dev, dev_priv->ips.fstart);
2937 mdelay(1);
2938 rgvswctl |= MEMCTL_CMD_STS;
2939 I915_WRITE(MEMSWCTL, rgvswctl);
2940 mdelay(1);
2941
2942 spin_unlock_irq(&mchdev_lock);
2943}
2944
2945/* There's a funny hw issue where the hw returns all 0 when reading from
2946 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
2947 * ourselves, instead of doing a rmw cycle (which might result in us clearing
2948 * all limits and the gpu stuck at whatever frequency it is at atm).
2949 */
2950static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
2951{
2952 u32 limits;
2953
2954 /* Only set the down limit when we've reached the lowest level to avoid
2955 * getting more interrupts, otherwise leave this clear. This prevents a
2956 * race in the hw when coming out of rc6: There's a tiny window where
2957 * the hw runs at the minimal clock before selecting the desired
2958 * frequency, if the down threshold expires in that window we will not
2959 * receive a down interrupt. */
2960 limits = dev_priv->rps.max_freq_softlimit << 24;
2961 if (val <= dev_priv->rps.min_freq_softlimit)
2962 limits |= dev_priv->rps.min_freq_softlimit << 16;
2963
2964 return limits;
2965}
2966
2967static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
2968{
2969 int new_power;
2970
2971 new_power = dev_priv->rps.power;
2972 switch (dev_priv->rps.power) {
2973 case LOW_POWER:
2974 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
2975 new_power = BETWEEN;
2976 break;
2977
2978 case BETWEEN:
2979 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
2980 new_power = LOW_POWER;
2981 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
2982 new_power = HIGH_POWER;
2983 break;
2984
2985 case HIGH_POWER:
2986 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
2987 new_power = BETWEEN;
2988 break;
2989 }
2990 /* Max/min bins are special */
2991 if (val == dev_priv->rps.min_freq_softlimit)
2992 new_power = LOW_POWER;
2993 if (val == dev_priv->rps.max_freq_softlimit)
2994 new_power = HIGH_POWER;
2995 if (new_power == dev_priv->rps.power)
2996 return;
2997
2998 /* Note the units here are not exactly 1us, but 1280ns. */
2999 switch (new_power) {
3000 case LOW_POWER:
3001 /* Upclock if more than 95% busy over 16ms */
3002 I915_WRITE(GEN6_RP_UP_EI, 12500);
3003 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
3004
3005 /* Downclock if less than 85% busy over 32ms */
3006 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3007 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
3008
3009 I915_WRITE(GEN6_RP_CONTROL,
3010 GEN6_RP_MEDIA_TURBO |
3011 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3012 GEN6_RP_MEDIA_IS_GFX |
3013 GEN6_RP_ENABLE |
3014 GEN6_RP_UP_BUSY_AVG |
3015 GEN6_RP_DOWN_IDLE_AVG);
3016 break;
3017
3018 case BETWEEN:
3019 /* Upclock if more than 90% busy over 13ms */
3020 I915_WRITE(GEN6_RP_UP_EI, 10250);
3021 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
3022
3023 /* Downclock if less than 75% busy over 32ms */
3024 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3025 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
3026
3027 I915_WRITE(GEN6_RP_CONTROL,
3028 GEN6_RP_MEDIA_TURBO |
3029 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3030 GEN6_RP_MEDIA_IS_GFX |
3031 GEN6_RP_ENABLE |
3032 GEN6_RP_UP_BUSY_AVG |
3033 GEN6_RP_DOWN_IDLE_AVG);
3034 break;
3035
3036 case HIGH_POWER:
3037 /* Upclock if more than 85% busy over 10ms */
3038 I915_WRITE(GEN6_RP_UP_EI, 8000);
3039 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
3040
3041 /* Downclock if less than 60% busy over 32ms */
3042 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3043 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
3044
3045 I915_WRITE(GEN6_RP_CONTROL,
3046 GEN6_RP_MEDIA_TURBO |
3047 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3048 GEN6_RP_MEDIA_IS_GFX |
3049 GEN6_RP_ENABLE |
3050 GEN6_RP_UP_BUSY_AVG |
3051 GEN6_RP_DOWN_IDLE_AVG);
3052 break;
3053 }
3054
3055 dev_priv->rps.power = new_power;
3056 dev_priv->rps.last_adj = 0;
3057}
3058
3059static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
3060{
3061 u32 mask = 0;
3062
3063 if (val > dev_priv->rps.min_freq_softlimit)
3064 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
3065 if (val < dev_priv->rps.max_freq_softlimit)
3066 mask |= GEN6_PM_RP_UP_THRESHOLD;
3067
3068 /* IVB and SNB hard hangs on looping batchbuffer
3069 * if GEN6_PM_UP_EI_EXPIRED is masked.
3070 */
3071 if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev))
3072 mask |= GEN6_PM_RP_UP_EI_EXPIRED;
3073
3074 return ~mask;
3075}
3076
3077/* gen6_set_rps is called to update the frequency request, but should also be
3078 * called when the range (min_delay and max_delay) is modified so that we can
3079 * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
3080void gen6_set_rps(struct drm_device *dev, u8 val)
3081{
3082 struct drm_i915_private *dev_priv = dev->dev_private;
3083
3084 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3085 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3086 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3087
3088 /* min/max delay may still have been modified so be sure to
3089 * write the limits value.
3090 */
3091 if (val != dev_priv->rps.cur_freq) {
3092 gen6_set_rps_thresholds(dev_priv, val);
3093
3094 if (IS_HASWELL(dev))
3095 I915_WRITE(GEN6_RPNSWREQ,
3096 HSW_FREQUENCY(val));
3097 else
3098 I915_WRITE(GEN6_RPNSWREQ,
3099 GEN6_FREQUENCY(val) |
3100 GEN6_OFFSET(0) |
3101 GEN6_AGGRESSIVE_TURBO);
3102 }
3103
3104 /* Make sure we continue to get interrupts
3105 * until we hit the minimum or maximum frequencies.
3106 */
3107 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
3108 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3109
3110 POSTING_READ(GEN6_RPNSWREQ);
3111
3112 dev_priv->rps.cur_freq = val;
3113 trace_intel_gpu_freq_change(val * 50);
3114}
3115
3116/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
3117 *
3118 * * If Gfx is Idle, then
3119 * 1. Mask Turbo interrupts
3120 * 2. Bring up Gfx clock
3121 * 3. Change the freq to Rpn and wait till P-Unit updates freq
3122 * 4. Clear the Force GFX CLK ON bit so that Gfx can down
3123 * 5. Unmask Turbo interrupts
3124*/
3125static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
3126{
3127 /*
3128 * When we are idle. Drop to min voltage state.
3129 */
3130
3131 if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
3132 return;
3133
3134 /* Mask turbo interrupt so that they will not come in between */
3135 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
3136
3137 /* Bring up the Gfx clock */
3138 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG,
3139 I915_READ(VLV_GTLC_SURVIVABILITY_REG) |
3140 VLV_GFX_CLK_FORCE_ON_BIT);
3141
3142 if (wait_for(((VLV_GFX_CLK_STATUS_BIT &
3143 I915_READ(VLV_GTLC_SURVIVABILITY_REG)) != 0), 5)) {
3144 DRM_ERROR("GFX_CLK_ON request timed out\n");
3145 return;
3146 }
3147
3148 dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
3149
3150 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
3151 dev_priv->rps.min_freq_softlimit);
3152
3153 if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
3154 & GENFREQSTATUS) == 0, 5))
3155 DRM_ERROR("timed out waiting for Punit\n");
3156
3157 /* Release the Gfx clock */
3158 I915_WRITE(VLV_GTLC_SURVIVABILITY_REG,
3159 I915_READ(VLV_GTLC_SURVIVABILITY_REG) &
3160 ~VLV_GFX_CLK_FORCE_ON_BIT);
3161
3162 I915_WRITE(GEN6_PMINTRMSK,
3163 gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
3164}
3165
3166void gen6_rps_idle(struct drm_i915_private *dev_priv)
3167{
3168 struct drm_device *dev = dev_priv->dev;
3169
3170 mutex_lock(&dev_priv->rps.hw_lock);
3171 if (dev_priv->rps.enabled) {
3172 if (IS_VALLEYVIEW(dev))
3173 vlv_set_rps_idle(dev_priv);
3174 else
3175 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3176 dev_priv->rps.last_adj = 0;
3177 }
3178 mutex_unlock(&dev_priv->rps.hw_lock);
3179}
3180
3181void gen6_rps_boost(struct drm_i915_private *dev_priv)
3182{
3183 struct drm_device *dev = dev_priv->dev;
3184
3185 mutex_lock(&dev_priv->rps.hw_lock);
3186 if (dev_priv->rps.enabled) {
3187 if (IS_VALLEYVIEW(dev))
3188 valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
3189 else
3190 gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
3191 dev_priv->rps.last_adj = 0;
3192 }
3193 mutex_unlock(&dev_priv->rps.hw_lock);
3194}
3195
3196void valleyview_set_rps(struct drm_device *dev, u8 val)
3197{
3198 struct drm_i915_private *dev_priv = dev->dev_private;
3199
3200 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3201 WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3202 WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3203
3204 DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
3205 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
3206 dev_priv->rps.cur_freq,
3207 vlv_gpu_freq(dev_priv, val), val);
3208
3209 if (val != dev_priv->rps.cur_freq)
3210 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3211
3212 I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3213
3214 dev_priv->rps.cur_freq = val;
3215 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
3216}
3217
3218static void gen6_disable_rps_interrupts(struct drm_device *dev)
3219{
3220 struct drm_i915_private *dev_priv = dev->dev_private;
3221
3222 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
3223 I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) &
3224 ~dev_priv->pm_rps_events);
3225 /* Complete PM interrupt masking here doesn't race with the rps work
3226 * item again unmasking PM interrupts because that is using a different
3227 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
3228 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
3229
3230 spin_lock_irq(&dev_priv->irq_lock);
3231 dev_priv->rps.pm_iir = 0;
3232 spin_unlock_irq(&dev_priv->irq_lock);
3233
3234 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
3235}
3236
3237static void gen6_disable_rps(struct drm_device *dev)
3238{
3239 struct drm_i915_private *dev_priv = dev->dev_private;
3240
3241 I915_WRITE(GEN6_RC_CONTROL, 0);
3242 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
3243
3244 gen6_disable_rps_interrupts(dev);
3245}
3246
3247static void valleyview_disable_rps(struct drm_device *dev)
3248{
3249 struct drm_i915_private *dev_priv = dev->dev_private;
3250
3251 I915_WRITE(GEN6_RC_CONTROL, 0);
3252
3253 gen6_disable_rps_interrupts(dev);
3254}
3255
3256static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
3257{
3258 DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
3259 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
3260 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
3261 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
3262}
3263
3264int intel_enable_rc6(const struct drm_device *dev)
3265{
3266 /* No RC6 before Ironlake */
3267 if (INTEL_INFO(dev)->gen < 5)
3268 return 0;
3269
3270 /* Respect the kernel parameter if it is set */
3271 if (i915.enable_rc6 >= 0)
3272 return i915.enable_rc6;
3273
3274 /* Disable RC6 on Ironlake */
3275 if (INTEL_INFO(dev)->gen == 5)
3276 return 0;
3277
3278 if (IS_IVYBRIDGE(dev))
3279 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
3280
3281 return INTEL_RC6_ENABLE;
3282}
3283
3284static void gen6_enable_rps_interrupts(struct drm_device *dev)
3285{
3286 struct drm_i915_private *dev_priv = dev->dev_private;
3287
3288 spin_lock_irq(&dev_priv->irq_lock);
3289 WARN_ON(dev_priv->rps.pm_iir);
3290 snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
3291 I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
3292 spin_unlock_irq(&dev_priv->irq_lock);
3293}
3294
3295static void gen8_enable_rps(struct drm_device *dev)
3296{
3297 struct drm_i915_private *dev_priv = dev->dev_private;
3298 struct intel_ring_buffer *ring;
3299 uint32_t rc6_mask = 0, rp_state_cap;
3300 int unused;
3301
3302 /* 1a: Software RC state - RC0 */
3303 I915_WRITE(GEN6_RC_STATE, 0);
3304
3305 /* 1c & 1d: Get forcewake during program sequence. Although the driver
3306 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
3307 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3308
3309 /* 2a: Disable RC states. */
3310 I915_WRITE(GEN6_RC_CONTROL, 0);
3311
3312 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3313
3314 /* 2b: Program RC6 thresholds.*/
3315 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
3316 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
3317 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
3318 for_each_ring(ring, dev_priv, unused)
3319 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3320 I915_WRITE(GEN6_RC_SLEEP, 0);
3321 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
3322
3323 /* 3: Enable RC6 */
3324 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
3325 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
3326 intel_print_rc6_info(dev, rc6_mask);
3327 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
3328 GEN6_RC_CTL_EI_MODE(1) |
3329 rc6_mask);
3330
3331 /* 4 Program defaults and thresholds for RPS*/
3332 I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
3333 I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */
3334 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
3335 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
3336
3337 /* Docs recommend 900MHz, and 300 MHz respectively */
3338 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
3339 dev_priv->rps.max_freq_softlimit << 24 |
3340 dev_priv->rps.min_freq_softlimit << 16);
3341
3342 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
3343 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
3344 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
3345 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
3346
3347 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3348
3349 /* 5: Enable RPS */
3350 I915_WRITE(GEN6_RP_CONTROL,
3351 GEN6_RP_MEDIA_TURBO |
3352 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3353 GEN6_RP_MEDIA_IS_GFX |
3354 GEN6_RP_ENABLE |
3355 GEN6_RP_UP_BUSY_AVG |
3356 GEN6_RP_DOWN_IDLE_AVG);
3357
3358 /* 6: Ring frequency + overclocking (our driver does this later */
3359
3360 gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
3361
3362 gen6_enable_rps_interrupts(dev);
3363
3364 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3365}
3366
3367static void gen6_enable_rps(struct drm_device *dev)
3368{
3369 struct drm_i915_private *dev_priv = dev->dev_private;
3370 struct intel_ring_buffer *ring;
3371 u32 rp_state_cap;
3372 u32 gt_perf_status;
3373 u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3374 u32 gtfifodbg;
3375 int rc6_mode;
3376 int i, ret;
3377
3378 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3379
3380 /* Here begins a magic sequence of register writes to enable
3381 * auto-downclocking.
3382 *
3383 * Perhaps there might be some value in exposing these to
3384 * userspace...
3385 */
3386 I915_WRITE(GEN6_RC_STATE, 0);
3387
3388 /* Clear the DBG now so we don't confuse earlier errors */
3389 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
3390 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
3391 I915_WRITE(GTFIFODBG, gtfifodbg);
3392 }
3393
3394 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3395
3396 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3397 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
3398
3399 /* All of these values are in units of 50MHz */
3400 dev_priv->rps.cur_freq = 0;
3401 /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */
3402 dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
3403 dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
3404 dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
3405 /* XXX: only BYT has a special efficient freq */
3406 dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
3407 /* hw_max = RP0 until we check for overclocking */
3408 dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
3409
3410 /* Preserve min/max settings in case of re-init */
3411 if (dev_priv->rps.max_freq_softlimit == 0)
3412 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
3413
3414 if (dev_priv->rps.min_freq_softlimit == 0)
3415 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
3416
3417 /* disable the counters and set deterministic thresholds */
3418 I915_WRITE(GEN6_RC_CONTROL, 0);
3419
3420 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
3421 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
3422 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
3423 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
3424 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
3425
3426 for_each_ring(ring, dev_priv, i)
3427 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3428
3429 I915_WRITE(GEN6_RC_SLEEP, 0);
3430 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
3431 if (IS_IVYBRIDGE(dev))
3432 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
3433 else
3434 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3435 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3436 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
3437
3438 /* Check if we are enabling RC6 */
3439 rc6_mode = intel_enable_rc6(dev_priv->dev);
3440 if (rc6_mode & INTEL_RC6_ENABLE)
3441 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
3442
3443 /* We don't use those on Haswell */
3444 if (!IS_HASWELL(dev)) {
3445 if (rc6_mode & INTEL_RC6p_ENABLE)
3446 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
3447
3448 if (rc6_mode & INTEL_RC6pp_ENABLE)
3449 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
3450 }
3451
3452 intel_print_rc6_info(dev, rc6_mask);
3453
3454 I915_WRITE(GEN6_RC_CONTROL,
3455 rc6_mask |
3456 GEN6_RC_CTL_EI_MODE(1) |
3457 GEN6_RC_CTL_HW_ENABLE);
3458
3459 /* Power down if completely idle for over 50ms */
3460 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3461 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3462
3463 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
3464 if (ret)
3465 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
3466
3467 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3468 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
3469 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
3470 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
3471 (pcu_mbox & 0xff) * 50);
3472 dev_priv->rps.max_freq = pcu_mbox & 0xff;
3473 }
3474
3475 dev_priv->rps.power = HIGH_POWER; /* force a reset */
3476 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3477
3478 gen6_enable_rps_interrupts(dev);
3479
3480 rc6vids = 0;
3481 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
3482 if (IS_GEN6(dev) && ret) {
3483 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
3484 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
3485 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
3486 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
3487 rc6vids &= 0xffff00;
3488 rc6vids |= GEN6_ENCODE_RC6_VID(450);
3489 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
3490 if (ret)
3491 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
3492 }
3493
3494 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3495}
3496
3497void gen6_update_ring_freq(struct drm_device *dev)
3498{
3499 struct drm_i915_private *dev_priv = dev->dev_private;
3500 int min_freq = 15;
3501 unsigned int gpu_freq;
3502 unsigned int max_ia_freq, min_ring_freq;
3503 int scaling_factor = 180;
3504 struct cpufreq_policy *policy;
3505
3506 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3507
3508 policy = cpufreq_cpu_get(0);
3509 if (policy) {
3510 max_ia_freq = policy->cpuinfo.max_freq;
3511 cpufreq_cpu_put(policy);
3512 } else {
3513 /*
3514 * Default to measured freq if none found, PCU will ensure we
3515 * don't go over
3516 */
3517 max_ia_freq = tsc_khz;
3518 }
3519
3520 /* Convert from kHz to MHz */
3521 max_ia_freq /= 1000;
3522
3523 min_ring_freq = I915_READ(DCLK) & 0xf;
3524 /* convert DDR frequency from units of 266.6MHz to bandwidth */
3525 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3526
3527 /*
3528 * For each potential GPU frequency, load a ring frequency we'd like
3529 * to use for memory access. We do this by specifying the IA frequency
3530 * the PCU should use as a reference to determine the ring frequency.
3531 */
3532 for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit;
3533 gpu_freq--) {
3534 int diff = dev_priv->rps.max_freq_softlimit - gpu_freq;
3535 unsigned int ia_freq = 0, ring_freq = 0;
3536
3537 if (INTEL_INFO(dev)->gen >= 8) {
3538 /* max(2 * GT, DDR). NB: GT is 50MHz units */
3539 ring_freq = max(min_ring_freq, gpu_freq);
3540 } else if (IS_HASWELL(dev)) {
3541 ring_freq = mult_frac(gpu_freq, 5, 4);
3542 ring_freq = max(min_ring_freq, ring_freq);
3543 /* leave ia_freq as the default, chosen by cpufreq */
3544 } else {
3545 /* On older processors, there is no separate ring
3546 * clock domain, so in order to boost the bandwidth
3547 * of the ring, we need to upclock the CPU (ia_freq).
3548 *
3549 * For GPU frequencies less than 750MHz,
3550 * just use the lowest ring freq.
3551 */
3552 if (gpu_freq < min_freq)
3553 ia_freq = 800;
3554 else
3555 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
3556 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3557 }
3558
3559 sandybridge_pcode_write(dev_priv,
3560 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3561 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
3562 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
3563 gpu_freq);
3564 }
3565}
3566
3567int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
3568{
3569 u32 val, rp0;
3570
3571 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
3572
3573 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
3574 /* Clamp to max */
3575 rp0 = min_t(u32, rp0, 0xea);
3576
3577 return rp0;
3578}
3579
3580static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
3581{
3582 u32 val, rpe;
3583
3584 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
3585 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
3586 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
3587 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
3588
3589 return rpe;
3590}
3591
3592int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
3593{
3594 return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
3595}
3596
3597/* Check that the pctx buffer wasn't move under us. */
3598static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
3599{
3600 unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
3601
3602 WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
3603 dev_priv->vlv_pctx->stolen->start);
3604}
3605
3606static void valleyview_setup_pctx(struct drm_device *dev)
3607{
3608 struct drm_i915_private *dev_priv = dev->dev_private;
3609 struct drm_i915_gem_object *pctx;
3610 unsigned long pctx_paddr;
3611 u32 pcbr;
3612 int pctx_size = 24*1024;
3613
3614 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3615
3616 pcbr = I915_READ(VLV_PCBR);
3617 if (pcbr) {
3618 /* BIOS set it up already, grab the pre-alloc'd space */
3619 int pcbr_offset;
3620
3621 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
3622 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
3623 pcbr_offset,
3624 I915_GTT_OFFSET_NONE,
3625 pctx_size);
3626 goto out;
3627 }
3628
3629 /*
3630 * From the Gunit register HAS:
3631 * The Gfx driver is expected to program this register and ensure
3632 * proper allocation within Gfx stolen memory. For example, this
3633 * register should be programmed such than the PCBR range does not
3634 * overlap with other ranges, such as the frame buffer, protected
3635 * memory, or any other relevant ranges.
3636 */
3637 pctx = i915_gem_object_create_stolen(dev, pctx_size);
3638 if (!pctx) {
3639 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
3640 return;
3641 }
3642
3643 pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
3644 I915_WRITE(VLV_PCBR, pctx_paddr);
3645
3646out:
3647 dev_priv->vlv_pctx = pctx;
3648}
3649
3650static void valleyview_cleanup_pctx(struct drm_device *dev)
3651{
3652 struct drm_i915_private *dev_priv = dev->dev_private;
3653
3654 if (WARN_ON(!dev_priv->vlv_pctx))
3655 return;
3656
3657 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
3658 dev_priv->vlv_pctx = NULL;
3659}
3660
3661static void valleyview_enable_rps(struct drm_device *dev)
3662{
3663 struct drm_i915_private *dev_priv = dev->dev_private;
3664 struct intel_ring_buffer *ring;
3665 u32 gtfifodbg, val, rc6_mode = 0;
3666 int i;
3667
3668 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3669
3670 valleyview_check_pctx(dev_priv);
3671
3672 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
3673 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
3674 gtfifodbg);
3675 I915_WRITE(GTFIFODBG, gtfifodbg);
3676 }
3677
3678 /* If VLV, Forcewake all wells, else re-direct to regular path */
3679 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3680
3681 I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
3682 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
3683 I915_WRITE(GEN6_RP_UP_EI, 66000);
3684 I915_WRITE(GEN6_RP_DOWN_EI, 350000);
3685
3686 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3687
3688 I915_WRITE(GEN6_RP_CONTROL,
3689 GEN6_RP_MEDIA_TURBO |
3690 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3691 GEN6_RP_MEDIA_IS_GFX |
3692 GEN6_RP_ENABLE |
3693 GEN6_RP_UP_BUSY_AVG |
3694 GEN6_RP_DOWN_IDLE_CONT);
3695
3696 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
3697 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
3698 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
3699
3700 for_each_ring(ring, dev_priv, i)
3701 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3702
3703 I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
3704
3705 /* allows RC6 residency counter to work */
3706 I915_WRITE(VLV_COUNTER_CONTROL,
3707 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
3708 VLV_MEDIA_RC6_COUNT_EN |
3709 VLV_RENDER_RC6_COUNT_EN));
3710 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
3711 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
3712
3713 intel_print_rc6_info(dev, rc6_mode);
3714
3715 I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
3716
3717 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
3718
3719 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
3720 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
3721
3722 dev_priv->rps.cur_freq = (val >> 8) & 0xff;
3723 DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
3724 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
3725 dev_priv->rps.cur_freq);
3726
3727 dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
3728 dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
3729 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
3730 vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
3731 dev_priv->rps.max_freq);
3732
3733 dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
3734 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
3735 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
3736 dev_priv->rps.efficient_freq);
3737
3738 dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
3739 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
3740 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
3741 dev_priv->rps.min_freq);
3742
3743 /* Preserve min/max settings in case of re-init */
3744 if (dev_priv->rps.max_freq_softlimit == 0)
3745 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
3746
3747 if (dev_priv->rps.min_freq_softlimit == 0)
3748 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
3749
3750 DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
3751 vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
3752 dev_priv->rps.efficient_freq);
3753
3754 valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
3755
3756 gen6_enable_rps_interrupts(dev);
3757
3758 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3759}
3760
3761void ironlake_teardown_rc6(struct drm_device *dev)
3762{
3763 struct drm_i915_private *dev_priv = dev->dev_private;
3764
3765 if (dev_priv->ips.renderctx) {
3766 i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
3767 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
3768 dev_priv->ips.renderctx = NULL;
3769 }
3770
3771 if (dev_priv->ips.pwrctx) {
3772 i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
3773 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
3774 dev_priv->ips.pwrctx = NULL;
3775 }
3776}
3777
3778static void ironlake_disable_rc6(struct drm_device *dev)
3779{
3780 struct drm_i915_private *dev_priv = dev->dev_private;
3781
3782 if (I915_READ(PWRCTXA)) {
3783 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
3784 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
3785 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
3786 50);
3787
3788 I915_WRITE(PWRCTXA, 0);
3789 POSTING_READ(PWRCTXA);
3790
3791 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
3792 POSTING_READ(RSTDBYCTL);
3793 }
3794}
3795
3796static int ironlake_setup_rc6(struct drm_device *dev)
3797{
3798 struct drm_i915_private *dev_priv = dev->dev_private;
3799
3800 if (dev_priv->ips.renderctx == NULL)
3801 dev_priv->ips.renderctx = intel_alloc_context_page(dev);
3802 if (!dev_priv->ips.renderctx)
3803 return -ENOMEM;
3804
3805 if (dev_priv->ips.pwrctx == NULL)
3806 dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
3807 if (!dev_priv->ips.pwrctx) {
3808 ironlake_teardown_rc6(dev);
3809 return -ENOMEM;
3810 }
3811
3812 return 0;
3813}
3814
3815static void ironlake_enable_rc6(struct drm_device *dev)
3816{
3817 struct drm_i915_private *dev_priv = dev->dev_private;
3818 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3819 bool was_interruptible;
3820 int ret;
3821
3822 /* rc6 disabled by default due to repeated reports of hanging during
3823 * boot and resume.
3824 */
3825 if (!intel_enable_rc6(dev))
3826 return;
3827
3828 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3829
3830 ret = ironlake_setup_rc6(dev);
3831 if (ret)
3832 return;
3833
3834 was_interruptible = dev_priv->mm.interruptible;
3835 dev_priv->mm.interruptible = false;
3836
3837 /*
3838 * GPU can automatically power down the render unit if given a page
3839 * to save state.
3840 */
3841 ret = intel_ring_begin(ring, 6);
3842 if (ret) {
3843 ironlake_teardown_rc6(dev);
3844 dev_priv->mm.interruptible = was_interruptible;
3845 return;
3846 }
3847
3848 intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
3849 intel_ring_emit(ring, MI_SET_CONTEXT);
3850 intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) |
3851 MI_MM_SPACE_GTT |
3852 MI_SAVE_EXT_STATE_EN |
3853 MI_RESTORE_EXT_STATE_EN |
3854 MI_RESTORE_INHIBIT);
3855 intel_ring_emit(ring, MI_SUSPEND_FLUSH);
3856 intel_ring_emit(ring, MI_NOOP);
3857 intel_ring_emit(ring, MI_FLUSH);
3858 intel_ring_advance(ring);
3859
3860 /*
3861 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
3862 * does an implicit flush, combined with MI_FLUSH above, it should be
3863 * safe to assume that renderctx is valid
3864 */
3865 ret = intel_ring_idle(ring);
3866 dev_priv->mm.interruptible = was_interruptible;
3867 if (ret) {
3868 DRM_ERROR("failed to enable ironlake power savings\n");
3869 ironlake_teardown_rc6(dev);
3870 return;
3871 }
3872
3873 I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN);
3874 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
3875
3876 intel_print_rc6_info(dev, INTEL_RC6_ENABLE);
3877}
3878
3879static unsigned long intel_pxfreq(u32 vidfreq)
3880{
3881 unsigned long freq;
3882 int div = (vidfreq & 0x3f0000) >> 16;
3883 int post = (vidfreq & 0x3000) >> 12;
3884 int pre = (vidfreq & 0x7);
3885
3886 if (!pre)
3887 return 0;
3888
3889 freq = ((div * 133333) / ((1<<post) * pre));
3890
3891 return freq;
3892}
3893
3894static const struct cparams {
3895 u16 i;
3896 u16 t;
3897 u16 m;
3898 u16 c;
3899} cparams[] = {
3900 { 1, 1333, 301, 28664 },
3901 { 1, 1066, 294, 24460 },
3902 { 1, 800, 294, 25192 },
3903 { 0, 1333, 276, 27605 },
3904 { 0, 1066, 276, 27605 },
3905 { 0, 800, 231, 23784 },
3906};
3907
3908static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
3909{
3910 u64 total_count, diff, ret;
3911 u32 count1, count2, count3, m = 0, c = 0;
3912 unsigned long now = jiffies_to_msecs(jiffies), diff1;
3913 int i;
3914
3915 assert_spin_locked(&mchdev_lock);
3916
3917 diff1 = now - dev_priv->ips.last_time1;
3918
3919 /* Prevent division-by-zero if we are asking too fast.
3920 * Also, we don't get interesting results if we are polling
3921 * faster than once in 10ms, so just return the saved value
3922 * in such cases.
3923 */
3924 if (diff1 <= 10)
3925 return dev_priv->ips.chipset_power;
3926
3927 count1 = I915_READ(DMIEC);
3928 count2 = I915_READ(DDREC);
3929 count3 = I915_READ(CSIEC);
3930
3931 total_count = count1 + count2 + count3;
3932
3933 /* FIXME: handle per-counter overflow */
3934 if (total_count < dev_priv->ips.last_count1) {
3935 diff = ~0UL - dev_priv->ips.last_count1;
3936 diff += total_count;
3937 } else {
3938 diff = total_count - dev_priv->ips.last_count1;
3939 }
3940
3941 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
3942 if (cparams[i].i == dev_priv->ips.c_m &&
3943 cparams[i].t == dev_priv->ips.r_t) {
3944 m = cparams[i].m;
3945 c = cparams[i].c;
3946 break;
3947 }
3948 }
3949
3950 diff = div_u64(diff, diff1);
3951 ret = ((m * diff) + c);
3952 ret = div_u64(ret, 10);
3953
3954 dev_priv->ips.last_count1 = total_count;
3955 dev_priv->ips.last_time1 = now;
3956
3957 dev_priv->ips.chipset_power = ret;
3958
3959 return ret;
3960}
3961
3962unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
3963{
3964 struct drm_device *dev = dev_priv->dev;
3965 unsigned long val;
3966
3967 if (INTEL_INFO(dev)->gen != 5)
3968 return 0;
3969
3970 spin_lock_irq(&mchdev_lock);
3971
3972 val = __i915_chipset_val(dev_priv);
3973
3974 spin_unlock_irq(&mchdev_lock);
3975
3976 return val;
3977}
3978
3979unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
3980{
3981 unsigned long m, x, b;
3982 u32 tsfs;
3983
3984 tsfs = I915_READ(TSFS);
3985
3986 m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
3987 x = I915_READ8(TR1);
3988
3989 b = tsfs & TSFS_INTR_MASK;
3990
3991 return ((m * x) / 127) - b;
3992}
3993
3994static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
3995{
3996 struct drm_device *dev = dev_priv->dev;
3997 static const struct v_table {
3998 u16 vd; /* in .1 mil */
3999 u16 vm; /* in .1 mil */
4000 } v_table[] = {
4001 { 0, 0, },
4002 { 375, 0, },
4003 { 500, 0, },
4004 { 625, 0, },
4005 { 750, 0, },
4006 { 875, 0, },
4007 { 1000, 0, },
4008 { 1125, 0, },
4009 { 4125, 3000, },
4010 { 4125, 3000, },
4011 { 4125, 3000, },
4012 { 4125, 3000, },
4013 { 4125, 3000, },
4014 { 4125, 3000, },
4015 { 4125, 3000, },
4016 { 4125, 3000, },
4017 { 4125, 3000, },
4018 { 4125, 3000, },
4019 { 4125, 3000, },
4020 { 4125, 3000, },
4021 { 4125, 3000, },
4022 { 4125, 3000, },
4023 { 4125, 3000, },
4024 { 4125, 3000, },
4025 { 4125, 3000, },
4026 { 4125, 3000, },
4027 { 4125, 3000, },
4028 { 4125, 3000, },
4029 { 4125, 3000, },
4030 { 4125, 3000, },
4031 { 4125, 3000, },
4032 { 4125, 3000, },
4033 { 4250, 3125, },
4034 { 4375, 3250, },
4035 { 4500, 3375, },
4036 { 4625, 3500, },
4037 { 4750, 3625, },
4038 { 4875, 3750, },
4039 { 5000, 3875, },
4040 { 5125, 4000, },
4041 { 5250, 4125, },
4042 { 5375, 4250, },
4043 { 5500, 4375, },
4044 { 5625, 4500, },
4045 { 5750, 4625, },
4046 { 5875, 4750, },
4047 { 6000, 4875, },
4048 { 6125, 5000, },
4049 { 6250, 5125, },
4050 { 6375, 5250, },
4051 { 6500, 5375, },
4052 { 6625, 5500, },
4053 { 6750, 5625, },
4054 { 6875, 5750, },
4055 { 7000, 5875, },
4056 { 7125, 6000, },
4057 { 7250, 6125, },
4058 { 7375, 6250, },
4059 { 7500, 6375, },
4060 { 7625, 6500, },
4061 { 7750, 6625, },
4062 { 7875, 6750, },
4063 { 8000, 6875, },
4064 { 8125, 7000, },
4065 { 8250, 7125, },
4066 { 8375, 7250, },
4067 { 8500, 7375, },
4068 { 8625, 7500, },
4069 { 8750, 7625, },
4070 { 8875, 7750, },
4071 { 9000, 7875, },
4072 { 9125, 8000, },
4073 { 9250, 8125, },
4074 { 9375, 8250, },
4075 { 9500, 8375, },
4076 { 9625, 8500, },
4077 { 9750, 8625, },
4078 { 9875, 8750, },
4079 { 10000, 8875, },
4080 { 10125, 9000, },
4081 { 10250, 9125, },
4082 { 10375, 9250, },
4083 { 10500, 9375, },
4084 { 10625, 9500, },
4085 { 10750, 9625, },
4086 { 10875, 9750, },
4087 { 11000, 9875, },
4088 { 11125, 10000, },
4089 { 11250, 10125, },
4090 { 11375, 10250, },
4091 { 11500, 10375, },
4092 { 11625, 10500, },
4093 { 11750, 10625, },
4094 { 11875, 10750, },
4095 { 12000, 10875, },
4096 { 12125, 11000, },
4097 { 12250, 11125, },
4098 { 12375, 11250, },
4099 { 12500, 11375, },
4100 { 12625, 11500, },
4101 { 12750, 11625, },
4102 { 12875, 11750, },
4103 { 13000, 11875, },
4104 { 13125, 12000, },
4105 { 13250, 12125, },
4106 { 13375, 12250, },
4107 { 13500, 12375, },
4108 { 13625, 12500, },
4109 { 13750, 12625, },
4110 { 13875, 12750, },
4111 { 14000, 12875, },
4112 { 14125, 13000, },
4113 { 14250, 13125, },
4114 { 14375, 13250, },
4115 { 14500, 13375, },
4116 { 14625, 13500, },
4117 { 14750, 13625, },
4118 { 14875, 13750, },
4119 { 15000, 13875, },
4120 { 15125, 14000, },
4121 { 15250, 14125, },
4122 { 15375, 14250, },
4123 { 15500, 14375, },
4124 { 15625, 14500, },
4125 { 15750, 14625, },
4126 { 15875, 14750, },
4127 { 16000, 14875, },
4128 { 16125, 15000, },
4129 };
4130 if (INTEL_INFO(dev)->is_mobile)
4131 return v_table[pxvid].vm;
4132 else
4133 return v_table[pxvid].vd;
4134}
4135
4136static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
4137{
4138 struct timespec now, diff1;
4139 u64 diff;
4140 unsigned long diffms;
4141 u32 count;
4142
4143 assert_spin_locked(&mchdev_lock);
4144
4145 getrawmonotonic(&now);
4146 diff1 = timespec_sub(now, dev_priv->ips.last_time2);
4147
4148 /* Don't divide by 0 */
4149 diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
4150 if (!diffms)
4151 return;
4152
4153 count = I915_READ(GFXEC);
4154
4155 if (count < dev_priv->ips.last_count2) {
4156 diff = ~0UL - dev_priv->ips.last_count2;
4157 diff += count;
4158 } else {
4159 diff = count - dev_priv->ips.last_count2;
4160 }
4161
4162 dev_priv->ips.last_count2 = count;
4163 dev_priv->ips.last_time2 = now;
4164
4165 /* More magic constants... */
4166 diff = diff * 1181;
4167 diff = div_u64(diff, diffms * 10);
4168 dev_priv->ips.gfx_power = diff;
4169}
4170
4171void i915_update_gfx_val(struct drm_i915_private *dev_priv)
4172{
4173 struct drm_device *dev = dev_priv->dev;
4174
4175 if (INTEL_INFO(dev)->gen != 5)
4176 return;
4177
4178 spin_lock_irq(&mchdev_lock);
4179
4180 __i915_update_gfx_val(dev_priv);
4181
4182 spin_unlock_irq(&mchdev_lock);
4183}
4184
4185static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
4186{
4187 unsigned long t, corr, state1, corr2, state2;
4188 u32 pxvid, ext_v;
4189
4190 assert_spin_locked(&mchdev_lock);
4191
4192 pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
4193 pxvid = (pxvid >> 24) & 0x7f;
4194 ext_v = pvid_to_extvid(dev_priv, pxvid);
4195
4196 state1 = ext_v;
4197
4198 t = i915_mch_val(dev_priv);
4199
4200 /* Revel in the empirically derived constants */
4201
4202 /* Correction factor in 1/100000 units */
4203 if (t > 80)
4204 corr = ((t * 2349) + 135940);
4205 else if (t >= 50)
4206 corr = ((t * 964) + 29317);
4207 else /* < 50 */
4208 corr = ((t * 301) + 1004);
4209
4210 corr = corr * ((150142 * state1) / 10000 - 78642);
4211 corr /= 100000;
4212 corr2 = (corr * dev_priv->ips.corr);
4213
4214 state2 = (corr2 * state1) / 10000;
4215 state2 /= 100; /* convert to mW */
4216
4217 __i915_update_gfx_val(dev_priv);
4218
4219 return dev_priv->ips.gfx_power + state2;
4220}
4221
4222unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
4223{
4224 struct drm_device *dev = dev_priv->dev;
4225 unsigned long val;
4226
4227 if (INTEL_INFO(dev)->gen != 5)
4228 return 0;
4229
4230 spin_lock_irq(&mchdev_lock);
4231
4232 val = __i915_gfx_val(dev_priv);
4233
4234 spin_unlock_irq(&mchdev_lock);
4235
4236 return val;
4237}
4238
4239/**
4240 * i915_read_mch_val - return value for IPS use
4241 *
4242 * Calculate and return a value for the IPS driver to use when deciding whether
4243 * we have thermal and power headroom to increase CPU or GPU power budget.
4244 */
4245unsigned long i915_read_mch_val(void)
4246{
4247 struct drm_i915_private *dev_priv;
4248 unsigned long chipset_val, graphics_val, ret = 0;
4249
4250 spin_lock_irq(&mchdev_lock);
4251 if (!i915_mch_dev)
4252 goto out_unlock;
4253 dev_priv = i915_mch_dev;
4254
4255 chipset_val = __i915_chipset_val(dev_priv);
4256 graphics_val = __i915_gfx_val(dev_priv);
4257
4258 ret = chipset_val + graphics_val;
4259
4260out_unlock:
4261 spin_unlock_irq(&mchdev_lock);
4262
4263 return ret;
4264}
4265EXPORT_SYMBOL_GPL(i915_read_mch_val);
4266
4267/**
4268 * i915_gpu_raise - raise GPU frequency limit
4269 *
4270 * Raise the limit; IPS indicates we have thermal headroom.
4271 */
4272bool i915_gpu_raise(void)
4273{
4274 struct drm_i915_private *dev_priv;
4275 bool ret = true;
4276
4277 spin_lock_irq(&mchdev_lock);
4278 if (!i915_mch_dev) {
4279 ret = false;
4280 goto out_unlock;
4281 }
4282 dev_priv = i915_mch_dev;
4283
4284 if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
4285 dev_priv->ips.max_delay--;
4286
4287out_unlock:
4288 spin_unlock_irq(&mchdev_lock);
4289
4290 return ret;
4291}
4292EXPORT_SYMBOL_GPL(i915_gpu_raise);
4293
4294/**
4295 * i915_gpu_lower - lower GPU frequency limit
4296 *
4297 * IPS indicates we're close to a thermal limit, so throttle back the GPU
4298 * frequency maximum.
4299 */
4300bool i915_gpu_lower(void)
4301{
4302 struct drm_i915_private *dev_priv;
4303 bool ret = true;
4304
4305 spin_lock_irq(&mchdev_lock);
4306 if (!i915_mch_dev) {
4307 ret = false;
4308 goto out_unlock;
4309 }
4310 dev_priv = i915_mch_dev;
4311
4312 if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
4313 dev_priv->ips.max_delay++;
4314
4315out_unlock:
4316 spin_unlock_irq(&mchdev_lock);
4317
4318 return ret;
4319}
4320EXPORT_SYMBOL_GPL(i915_gpu_lower);
4321
4322/**
4323 * i915_gpu_busy - indicate GPU business to IPS
4324 *
4325 * Tell the IPS driver whether or not the GPU is busy.
4326 */
4327bool i915_gpu_busy(void)
4328{
4329 struct drm_i915_private *dev_priv;
4330 struct intel_ring_buffer *ring;
4331 bool ret = false;
4332 int i;
4333
4334 spin_lock_irq(&mchdev_lock);
4335 if (!i915_mch_dev)
4336 goto out_unlock;
4337 dev_priv = i915_mch_dev;
4338
4339 for_each_ring(ring, dev_priv, i)
4340 ret |= !list_empty(&ring->request_list);
4341
4342out_unlock:
4343 spin_unlock_irq(&mchdev_lock);
4344
4345 return ret;
4346}
4347EXPORT_SYMBOL_GPL(i915_gpu_busy);
4348
4349/**
4350 * i915_gpu_turbo_disable - disable graphics turbo
4351 *
4352 * Disable graphics turbo by resetting the max frequency and setting the
4353 * current frequency to the default.
4354 */
4355bool i915_gpu_turbo_disable(void)
4356{
4357 struct drm_i915_private *dev_priv;
4358 bool ret = true;
4359
4360 spin_lock_irq(&mchdev_lock);
4361 if (!i915_mch_dev) {
4362 ret = false;
4363 goto out_unlock;
4364 }
4365 dev_priv = i915_mch_dev;
4366
4367 dev_priv->ips.max_delay = dev_priv->ips.fstart;
4368
4369 if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
4370 ret = false;
4371
4372out_unlock:
4373 spin_unlock_irq(&mchdev_lock);
4374
4375 return ret;
4376}
4377EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
4378
4379/**
4380 * Tells the intel_ips driver that the i915 driver is now loaded, if
4381 * IPS got loaded first.
4382 *
4383 * This awkward dance is so that neither module has to depend on the
4384 * other in order for IPS to do the appropriate communication of
4385 * GPU turbo limits to i915.
4386 */
4387static void
4388ips_ping_for_i915_load(void)
4389{
4390 void (*link)(void);
4391
4392 link = symbol_get(ips_link_to_i915_driver);
4393 if (link) {
4394 link();
4395 symbol_put(ips_link_to_i915_driver);
4396 }
4397}
4398
4399void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
4400{
4401 /* We only register the i915 ips part with intel-ips once everything is
4402 * set up, to avoid intel-ips sneaking in and reading bogus values. */
4403 spin_lock_irq(&mchdev_lock);
4404 i915_mch_dev = dev_priv;
4405 spin_unlock_irq(&mchdev_lock);
4406
4407 ips_ping_for_i915_load();
4408}
4409
4410void intel_gpu_ips_teardown(void)
4411{
4412 spin_lock_irq(&mchdev_lock);
4413 i915_mch_dev = NULL;
4414 spin_unlock_irq(&mchdev_lock);
4415}
4416
4417static void intel_init_emon(struct drm_device *dev)
4418{
4419 struct drm_i915_private *dev_priv = dev->dev_private;
4420 u32 lcfuse;
4421 u8 pxw[16];
4422 int i;
4423
4424 /* Disable to program */
4425 I915_WRITE(ECR, 0);
4426 POSTING_READ(ECR);
4427
4428 /* Program energy weights for various events */
4429 I915_WRITE(SDEW, 0x15040d00);
4430 I915_WRITE(CSIEW0, 0x007f0000);
4431 I915_WRITE(CSIEW1, 0x1e220004);
4432 I915_WRITE(CSIEW2, 0x04000004);
4433
4434 for (i = 0; i < 5; i++)
4435 I915_WRITE(PEW + (i * 4), 0);
4436 for (i = 0; i < 3; i++)
4437 I915_WRITE(DEW + (i * 4), 0);
4438
4439 /* Program P-state weights to account for frequency power adjustment */
4440 for (i = 0; i < 16; i++) {
4441 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
4442 unsigned long freq = intel_pxfreq(pxvidfreq);
4443 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
4444 PXVFREQ_PX_SHIFT;
4445 unsigned long val;
4446
4447 val = vid * vid;
4448 val *= (freq / 1000);
4449 val *= 255;
4450 val /= (127*127*900);
4451 if (val > 0xff)
4452 DRM_ERROR("bad pxval: %ld\n", val);
4453 pxw[i] = val;
4454 }
4455 /* Render standby states get 0 weight */
4456 pxw[14] = 0;
4457 pxw[15] = 0;
4458
4459 for (i = 0; i < 4; i++) {
4460 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
4461 (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
4462 I915_WRITE(PXW + (i * 4), val);
4463 }
4464
4465 /* Adjust magic regs to magic values (more experimental results) */
4466 I915_WRITE(OGW0, 0);
4467 I915_WRITE(OGW1, 0);
4468 I915_WRITE(EG0, 0x00007f00);
4469 I915_WRITE(EG1, 0x0000000e);
4470 I915_WRITE(EG2, 0x000e0000);
4471 I915_WRITE(EG3, 0x68000300);
4472 I915_WRITE(EG4, 0x42000000);
4473 I915_WRITE(EG5, 0x00140031);
4474 I915_WRITE(EG6, 0);
4475 I915_WRITE(EG7, 0);
4476
4477 for (i = 0; i < 8; i++)
4478 I915_WRITE(PXWL + (i * 4), 0);
4479
4480 /* Enable PMON + select events */
4481 I915_WRITE(ECR, 0x80000019);
4482
4483 lcfuse = I915_READ(LCFUSE02);
4484
4485 dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
4486}
4487
4488void intel_init_gt_powersave(struct drm_device *dev)
4489{
4490 if (IS_VALLEYVIEW(dev))
4491 valleyview_setup_pctx(dev);
4492}
4493
4494void intel_cleanup_gt_powersave(struct drm_device *dev)
4495{
4496 if (IS_VALLEYVIEW(dev))
4497 valleyview_cleanup_pctx(dev);
4498}
4499
4500void intel_disable_gt_powersave(struct drm_device *dev)
4501{
4502 struct drm_i915_private *dev_priv = dev->dev_private;
4503
4504 /* Interrupts should be disabled already to avoid re-arming. */
4505 WARN_ON(dev->irq_enabled);
4506
4507 if (IS_IRONLAKE_M(dev)) {
4508 ironlake_disable_drps(dev);
4509 ironlake_disable_rc6(dev);
4510 } else if (INTEL_INFO(dev)->gen >= 6) {
4511 cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
4512 cancel_work_sync(&dev_priv->rps.work);
4513 mutex_lock(&dev_priv->rps.hw_lock);
4514 if (IS_VALLEYVIEW(dev))
4515 valleyview_disable_rps(dev);
4516 else
4517 gen6_disable_rps(dev);
4518 dev_priv->rps.enabled = false;
4519 mutex_unlock(&dev_priv->rps.hw_lock);
4520 }
4521}
4522
4523static void intel_gen6_powersave_work(struct work_struct *work)
4524{
4525 struct drm_i915_private *dev_priv =
4526 container_of(work, struct drm_i915_private,
4527 rps.delayed_resume_work.work);
4528 struct drm_device *dev = dev_priv->dev;
4529
4530 mutex_lock(&dev_priv->rps.hw_lock);
4531
4532 if (IS_VALLEYVIEW(dev)) {
4533 valleyview_enable_rps(dev);
4534 } else if (IS_BROADWELL(dev)) {
4535 gen8_enable_rps(dev);
4536 gen6_update_ring_freq(dev);
4537 } else {
4538 gen6_enable_rps(dev);
4539 gen6_update_ring_freq(dev);
4540 }
4541 dev_priv->rps.enabled = true;
4542 mutex_unlock(&dev_priv->rps.hw_lock);
4543}
4544
4545void intel_enable_gt_powersave(struct drm_device *dev)
4546{
4547 struct drm_i915_private *dev_priv = dev->dev_private;
4548
4549 if (IS_IRONLAKE_M(dev)) {
4550 ironlake_enable_drps(dev);
4551 ironlake_enable_rc6(dev);
4552 intel_init_emon(dev);
4553 } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
4554 /*
4555 * PCU communication is slow and this doesn't need to be
4556 * done at any specific time, so do this out of our fast path
4557 * to make resume and init faster.
4558 */
4559 schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
4560 round_jiffies_up_relative(HZ));
4561 }
4562}
4563
4564static void ibx_init_clock_gating(struct drm_device *dev)
4565{
4566 struct drm_i915_private *dev_priv = dev->dev_private;
4567
4568 /*
4569 * On Ibex Peak and Cougar Point, we need to disable clock
4570 * gating for the panel power sequencer or it will fail to
4571 * start up when no ports are active.
4572 */
4573 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4574}
4575
4576static void g4x_disable_trickle_feed(struct drm_device *dev)
4577{
4578 struct drm_i915_private *dev_priv = dev->dev_private;
4579 int pipe;
4580
4581 for_each_pipe(pipe) {
4582 I915_WRITE(DSPCNTR(pipe),
4583 I915_READ(DSPCNTR(pipe)) |
4584 DISPPLANE_TRICKLE_FEED_DISABLE);
4585 intel_flush_primary_plane(dev_priv, pipe);
4586 }
4587}
4588
4589static void ilk_init_lp_watermarks(struct drm_device *dev)
4590{
4591 struct drm_i915_private *dev_priv = dev->dev_private;
4592
4593 I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
4594 I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
4595 I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
4596
4597 /*
4598 * Don't touch WM1S_LP_EN here.
4599 * Doing so could cause underruns.
4600 */
4601}
4602
4603static void ironlake_init_clock_gating(struct drm_device *dev)
4604{
4605 struct drm_i915_private *dev_priv = dev->dev_private;
4606 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4607
4608 /*
4609 * Required for FBC
4610 * WaFbcDisableDpfcClockGating:ilk
4611 */
4612 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4613 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4614 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4615
4616 I915_WRITE(PCH_3DCGDIS0,
4617 MARIUNIT_CLOCK_GATE_DISABLE |
4618 SVSMUNIT_CLOCK_GATE_DISABLE);
4619 I915_WRITE(PCH_3DCGDIS1,
4620 VFMUNIT_CLOCK_GATE_DISABLE);
4621
4622 /*
4623 * According to the spec the following bits should be set in
4624 * order to enable memory self-refresh
4625 * The bit 22/21 of 0x42004
4626 * The bit 5 of 0x42020
4627 * The bit 15 of 0x45000
4628 */
4629 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4630 (I915_READ(ILK_DISPLAY_CHICKEN2) |
4631 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4632 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4633 I915_WRITE(DISP_ARB_CTL,
4634 (I915_READ(DISP_ARB_CTL) |
4635 DISP_FBC_WM_DIS));
4636
4637 ilk_init_lp_watermarks(dev);
4638
4639 /*
4640 * Based on the document from hardware guys the following bits
4641 * should be set unconditionally in order to enable FBC.
4642 * The bit 22 of 0x42000
4643 * The bit 22 of 0x42004
4644 * The bit 7,8,9 of 0x42020.
4645 */
4646 if (IS_IRONLAKE_M(dev)) {
4647 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4648 I915_WRITE(ILK_DISPLAY_CHICKEN1,
4649 I915_READ(ILK_DISPLAY_CHICKEN1) |
4650 ILK_FBCQ_DIS);
4651 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4652 I915_READ(ILK_DISPLAY_CHICKEN2) |
4653 ILK_DPARB_GATE);
4654 }
4655
4656 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
4657
4658 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4659 I915_READ(ILK_DISPLAY_CHICKEN2) |
4660 ILK_ELPIN_409_SELECT);
4661 I915_WRITE(_3D_CHICKEN2,
4662 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
4663 _3D_CHICKEN2_WM_READ_PIPELINED);
4664
4665 /* WaDisableRenderCachePipelinedFlush:ilk */
4666 I915_WRITE(CACHE_MODE_0,
4667 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4668
4669 g4x_disable_trickle_feed(dev);
4670
4671 ibx_init_clock_gating(dev);
4672}
4673
4674static void cpt_init_clock_gating(struct drm_device *dev)
4675{
4676 struct drm_i915_private *dev_priv = dev->dev_private;
4677 int pipe;
4678 uint32_t val;
4679
4680 /*
4681 * On Ibex Peak and Cougar Point, we need to disable clock
4682 * gating for the panel power sequencer or it will fail to
4683 * start up when no ports are active.
4684 */
4685 I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4686 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4687 PCH_CPUNIT_CLOCK_GATE_DISABLE);
4688 I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
4689 DPLS_EDP_PPS_FIX_DIS);
4690 /* The below fixes the weird display corruption, a few pixels shifted
4691 * downward, on (only) LVDS of some HP laptops with IVY.
4692 */
4693 for_each_pipe(pipe) {
4694 val = I915_READ(TRANS_CHICKEN2(pipe));
4695 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4696 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4697 if (dev_priv->vbt.fdi_rx_polarity_inverted)
4698 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4699 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
4700 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4701 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4702 I915_WRITE(TRANS_CHICKEN2(pipe), val);
4703 }
4704 /* WADP0ClockGatingDisable */
4705 for_each_pipe(pipe) {
4706 I915_WRITE(TRANS_CHICKEN1(pipe),
4707 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4708 }
4709}
4710
4711static void gen6_check_mch_setup(struct drm_device *dev)
4712{
4713 struct drm_i915_private *dev_priv = dev->dev_private;
4714 uint32_t tmp;
4715
4716 tmp = I915_READ(MCH_SSKPD);
4717 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
4718 DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
4719 DRM_INFO("This can cause pipe underruns and display issues.\n");
4720 DRM_INFO("Please upgrade your BIOS to fix this.\n");
4721 }
4722}
4723
4724static void gen6_init_clock_gating(struct drm_device *dev)
4725{
4726 struct drm_i915_private *dev_priv = dev->dev_private;
4727 uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4728
4729 I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
4730
4731 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4732 I915_READ(ILK_DISPLAY_CHICKEN2) |
4733 ILK_ELPIN_409_SELECT);
4734
4735 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
4736 I915_WRITE(_3D_CHICKEN,
4737 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
4738
4739 /* WaSetupGtModeTdRowDispatch:snb */
4740 if (IS_SNB_GT1(dev))
4741 I915_WRITE(GEN6_GT_MODE,
4742 _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
4743
4744 /*
4745 * BSpec recoomends 8x4 when MSAA is used,
4746 * however in practice 16x4 seems fastest.
4747 *
4748 * Note that PS/WM thread counts depend on the WIZ hashing
4749 * disable bit, which we don't touch here, but it's good
4750 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
4751 */
4752 I915_WRITE(GEN6_GT_MODE,
4753 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
4754
4755 ilk_init_lp_watermarks(dev);
4756
4757 I915_WRITE(CACHE_MODE_0,
4758 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
4759
4760 I915_WRITE(GEN6_UCGCTL1,
4761 I915_READ(GEN6_UCGCTL1) |
4762 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4763 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4764
4765 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4766 * gating disable must be set. Failure to set it results in
4767 * flickering pixels due to Z write ordering failures after
4768 * some amount of runtime in the Mesa "fire" demo, and Unigine
4769 * Sanctuary and Tropics, and apparently anything else with
4770 * alpha test or pixel discard.
4771 *
4772 * According to the spec, bit 11 (RCCUNIT) must also be set,
4773 * but we didn't debug actual testcases to find it out.
4774 *
4775 * WaDisableRCCUnitClockGating:snb
4776 * WaDisableRCPBUnitClockGating:snb
4777 */
4778 I915_WRITE(GEN6_UCGCTL2,
4779 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4780 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4781
4782 /* WaStripsFansDisableFastClipPerformanceFix:snb */
4783 I915_WRITE(_3D_CHICKEN3,
4784 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
4785
4786 /*
4787 * Bspec says:
4788 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
4789 * 3DSTATE_SF number of SF output attributes is more than 16."
4790 */
4791 I915_WRITE(_3D_CHICKEN3,
4792 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
4793
4794 /*
4795 * According to the spec the following bits should be
4796 * set in order to enable memory self-refresh and fbc:
4797 * The bit21 and bit22 of 0x42000
4798 * The bit21 and bit22 of 0x42004
4799 * The bit5 and bit7 of 0x42020
4800 * The bit14 of 0x70180
4801 * The bit14 of 0x71180
4802 *
4803 * WaFbcAsynchFlipDisableFbcQueue:snb
4804 */
4805 I915_WRITE(ILK_DISPLAY_CHICKEN1,
4806 I915_READ(ILK_DISPLAY_CHICKEN1) |
4807 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4808 I915_WRITE(ILK_DISPLAY_CHICKEN2,
4809 I915_READ(ILK_DISPLAY_CHICKEN2) |
4810 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4811 I915_WRITE(ILK_DSPCLK_GATE_D,
4812 I915_READ(ILK_DSPCLK_GATE_D) |
4813 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4814 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4815
4816 g4x_disable_trickle_feed(dev);
4817
4818 cpt_init_clock_gating(dev);
4819
4820 gen6_check_mch_setup(dev);
4821}
4822
4823static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
4824{
4825 uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
4826
4827 /*
4828 * WaVSThreadDispatchOverride:ivb,vlv
4829 *
4830 * This actually overrides the dispatch
4831 * mode for all thread types.
4832 */
4833 reg &= ~GEN7_FF_SCHED_MASK;
4834 reg |= GEN7_FF_TS_SCHED_HW;
4835 reg |= GEN7_FF_VS_SCHED_HW;
4836 reg |= GEN7_FF_DS_SCHED_HW;
4837
4838 I915_WRITE(GEN7_FF_THREAD_MODE, reg);
4839}
4840
4841static void lpt_init_clock_gating(struct drm_device *dev)
4842{
4843 struct drm_i915_private *dev_priv = dev->dev_private;
4844
4845 /*
4846 * TODO: this bit should only be enabled when really needed, then
4847 * disabled when not needed anymore in order to save power.
4848 */
4849 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
4850 I915_WRITE(SOUTH_DSPCLK_GATE_D,
4851 I915_READ(SOUTH_DSPCLK_GATE_D) |
4852 PCH_LP_PARTITION_LEVEL_DISABLE);
4853
4854 /* WADPOClockGatingDisable:hsw */
4855 I915_WRITE(_TRANSA_CHICKEN1,
4856 I915_READ(_TRANSA_CHICKEN1) |
4857 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4858}
4859
4860static void lpt_suspend_hw(struct drm_device *dev)
4861{
4862 struct drm_i915_private *dev_priv = dev->dev_private;
4863
4864 if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
4865 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
4866
4867 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4868 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
4869 }
4870}
4871
4872static void gen8_init_clock_gating(struct drm_device *dev)
4873{
4874 struct drm_i915_private *dev_priv = dev->dev_private;
4875 enum pipe pipe;
4876
4877 I915_WRITE(WM3_LP_ILK, 0);
4878 I915_WRITE(WM2_LP_ILK, 0);
4879 I915_WRITE(WM1_LP_ILK, 0);
4880
4881 /* FIXME(BDW): Check all the w/a, some might only apply to
4882 * pre-production hw. */
4883
4884 /* WaDisablePartialInstShootdown:bdw */
4885 I915_WRITE(GEN8_ROW_CHICKEN,
4886 _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
4887
4888 /* WaDisableThreadStallDopClockGating:bdw */
4889 /* FIXME: Unclear whether we really need this on production bdw. */
4890 I915_WRITE(GEN8_ROW_CHICKEN,
4891 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
4892
4893 /*
4894 * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
4895 * pre-production hardware
4896 */
4897 I915_WRITE(HALF_SLICE_CHICKEN3,
4898 _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
4899 I915_WRITE(HALF_SLICE_CHICKEN3,
4900 _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
4901 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
4902
4903 I915_WRITE(_3D_CHICKEN3,
4904 _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2));
4905
4906 I915_WRITE(COMMON_SLICE_CHICKEN2,
4907 _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
4908
4909 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
4910 _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
4911
4912 /* WaSwitchSolVfFArbitrationPriority:bdw */
4913 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
4914
4915 /* WaPsrDPAMaskVBlankInSRD:bdw */
4916 I915_WRITE(CHICKEN_PAR1_1,
4917 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
4918
4919 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4920 for_each_pipe(pipe) {
4921 I915_WRITE(CHICKEN_PIPESL_1(pipe),
4922 I915_READ(CHICKEN_PIPESL_1(pipe)) |
4923 BDW_DPRS_MASK_VBLANK_SRD);
4924 }
4925
4926 /* Use Force Non-Coherent whenever executing a 3D context. This is a
4927 * workaround for for a possible hang in the unlikely event a TLB
4928 * invalidation occurs during a PSD flush.
4929 */
4930 I915_WRITE(HDC_CHICKEN0,
4931 I915_READ(HDC_CHICKEN0) |
4932 _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
4933
4934 /* WaVSRefCountFullforceMissDisable:bdw */
4935 /* WaDSRefCountFullforceMissDisable:bdw */
4936 I915_WRITE(GEN7_FF_THREAD_MODE,
4937 I915_READ(GEN7_FF_THREAD_MODE) &
4938 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
4939
4940 /*
4941 * BSpec recommends 8x4 when MSAA is used,
4942 * however in practice 16x4 seems fastest.
4943 *
4944 * Note that PS/WM thread counts depend on the WIZ hashing
4945 * disable bit, which we don't touch here, but it's good
4946 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
4947 */
4948 I915_WRITE(GEN7_GT_MODE,
4949 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
4950
4951 I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
4952 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4953
4954 /* WaDisableSDEUnitClockGating:bdw */
4955 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
4956 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4957
4958 /* Wa4x4STCOptimizationDisable:bdw */
4959 I915_WRITE(CACHE_MODE_1,
4960 _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
4961}
4962
4963static void haswell_init_clock_gating(struct drm_device *dev)
4964{
4965 struct drm_i915_private *dev_priv = dev->dev_private;
4966
4967 ilk_init_lp_watermarks(dev);
4968
4969 /* L3 caching of data atomics doesn't work -- disable it. */
4970 I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
4971 I915_WRITE(HSW_ROW_CHICKEN3,
4972 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
4973
4974 /* This is required by WaCatErrorRejectionIssue:hsw */
4975 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4976 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4977 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4978
4979 /* WaVSRefCountFullforceMissDisable:hsw */
4980 I915_WRITE(GEN7_FF_THREAD_MODE,
4981 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
4982
4983 /* enable HiZ Raw Stall Optimization */
4984 I915_WRITE(CACHE_MODE_0_GEN7,
4985 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
4986
4987 /* WaDisable4x2SubspanOptimization:hsw */
4988 I915_WRITE(CACHE_MODE_1,
4989 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
4990
4991 /*
4992 * BSpec recommends 8x4 when MSAA is used,
4993 * however in practice 16x4 seems fastest.
4994 *
4995 * Note that PS/WM thread counts depend on the WIZ hashing
4996 * disable bit, which we don't touch here, but it's good
4997 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
4998 */
4999 I915_WRITE(GEN7_GT_MODE,
5000 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
5001
5002 /* WaSwitchSolVfFArbitrationPriority:hsw */
5003 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
5004
5005 /* WaRsPkgCStateDisplayPMReq:hsw */
5006 I915_WRITE(CHICKEN_PAR1_1,
5007 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
5008
5009 lpt_init_clock_gating(dev);
5010}
5011
5012static void ivybridge_init_clock_gating(struct drm_device *dev)
5013{
5014 struct drm_i915_private *dev_priv = dev->dev_private;
5015 uint32_t snpcr;
5016
5017 ilk_init_lp_watermarks(dev);
5018
5019 I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
5020
5021 /* WaDisableEarlyCull:ivb */
5022 I915_WRITE(_3D_CHICKEN3,
5023 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5024
5025 /* WaDisableBackToBackFlipFix:ivb */
5026 I915_WRITE(IVB_CHICKEN3,
5027 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5028 CHICKEN3_DGMG_DONE_FIX_DISABLE);
5029
5030 /* WaDisablePSDDualDispatchEnable:ivb */
5031 if (IS_IVB_GT1(dev))
5032 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5033 _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
5034
5035 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
5036 I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
5037 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
5038
5039 /* WaApplyL3ControlAndL3ChickenMode:ivb */
5040 I915_WRITE(GEN7_L3CNTLREG1,
5041 GEN7_WA_FOR_GEN7_L3_CONTROL);
5042 I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
5043 GEN7_WA_L3_CHICKEN_MODE);
5044 if (IS_IVB_GT1(dev))
5045 I915_WRITE(GEN7_ROW_CHICKEN2,
5046 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5047 else {
5048 /* must write both registers */
5049 I915_WRITE(GEN7_ROW_CHICKEN2,
5050 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5051 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
5052 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5053 }
5054
5055 /* WaForceL3Serialization:ivb */
5056 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5057 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5058
5059 /*
5060 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
5061 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
5062 */
5063 I915_WRITE(GEN6_UCGCTL2,
5064 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
5065
5066 /* This is required by WaCatErrorRejectionIssue:ivb */
5067 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5068 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5069 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5070
5071 g4x_disable_trickle_feed(dev);
5072
5073 gen7_setup_fixed_func_scheduler(dev_priv);
5074
5075 if (0) { /* causes HiZ corruption on ivb:gt1 */
5076 /* enable HiZ Raw Stall Optimization */
5077 I915_WRITE(CACHE_MODE_0_GEN7,
5078 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
5079 }
5080
5081 /* WaDisable4x2SubspanOptimization:ivb */
5082 I915_WRITE(CACHE_MODE_1,
5083 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
5084
5085 /*
5086 * BSpec recommends 8x4 when MSAA is used,
5087 * however in practice 16x4 seems fastest.
5088 *
5089 * Note that PS/WM thread counts depend on the WIZ hashing
5090 * disable bit, which we don't touch here, but it's good
5091 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
5092 */
5093 I915_WRITE(GEN7_GT_MODE,
5094 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
5095
5096 snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
5097 snpcr &= ~GEN6_MBC_SNPCR_MASK;
5098 snpcr |= GEN6_MBC_SNPCR_MED;
5099 I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
5100
5101 if (!HAS_PCH_NOP(dev))
5102 cpt_init_clock_gating(dev);
5103
5104 gen6_check_mch_setup(dev);
5105}
5106
5107static void valleyview_init_clock_gating(struct drm_device *dev)
5108{
5109 struct drm_i915_private *dev_priv = dev->dev_private;
5110 u32 val;
5111
5112 mutex_lock(&dev_priv->rps.hw_lock);
5113 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5114 mutex_unlock(&dev_priv->rps.hw_lock);
5115 switch ((val >> 6) & 3) {
5116 case 0:
5117 case 1:
5118 dev_priv->mem_freq = 800;
5119 break;
5120 case 2:
5121 dev_priv->mem_freq = 1066;
5122 break;
5123 case 3:
5124 dev_priv->mem_freq = 1333;
5125 break;
5126 }
5127 DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
5128
5129 I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
5130
5131 /* WaDisableEarlyCull:vlv */
5132 I915_WRITE(_3D_CHICKEN3,
5133 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5134
5135 /* WaDisableBackToBackFlipFix:vlv */
5136 I915_WRITE(IVB_CHICKEN3,
5137 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5138 CHICKEN3_DGMG_DONE_FIX_DISABLE);
5139
5140 /* WaPsdDispatchEnable:vlv */
5141 /* WaDisablePSDDualDispatchEnable:vlv */
5142 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5143 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
5144 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
5145
5146 /* WaForceL3Serialization:vlv */
5147 I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5148 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5149
5150 /* WaDisableDopClockGating:vlv */
5151 I915_WRITE(GEN7_ROW_CHICKEN2,
5152 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5153
5154 /* This is required by WaCatErrorRejectionIssue:vlv */
5155 I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5156 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5157 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5158
5159 gen7_setup_fixed_func_scheduler(dev_priv);
5160
5161 /*
5162 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
5163 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
5164 */
5165 I915_WRITE(GEN6_UCGCTL2,
5166 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
5167
5168 /* WaDisableL3Bank2xClockGate:vlv */
5169 I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
5170
5171 I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
5172
5173 /*
5174 * BSpec says this must be set, even though
5175 * WaDisable4x2SubspanOptimization isn't listed for VLV.
5176 */
5177 I915_WRITE(CACHE_MODE_1,
5178 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
5179
5180 /*
5181 * WaIncreaseL3CreditsForVLVB0:vlv
5182 * This is the hardware default actually.
5183 */
5184 I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
5185
5186 /*
5187 * WaDisableVLVClockGating_VBIIssue:vlv
5188 * Disable clock gating on th GCFG unit to prevent a delay
5189 * in the reporting of vblank events.
5190 */
5191 I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
5192}
5193
5194static void g4x_init_clock_gating(struct drm_device *dev)
5195{
5196 struct drm_i915_private *dev_priv = dev->dev_private;
5197 uint32_t dspclk_gate;
5198
5199 I915_WRITE(RENCLK_GATE_D1, 0);
5200 I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
5201 GS_UNIT_CLOCK_GATE_DISABLE |
5202 CL_UNIT_CLOCK_GATE_DISABLE);
5203 I915_WRITE(RAMCLK_GATE_D, 0);
5204 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
5205 OVRUNIT_CLOCK_GATE_DISABLE |
5206 OVCUNIT_CLOCK_GATE_DISABLE;
5207 if (IS_GM45(dev))
5208 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
5209 I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
5210
5211 /* WaDisableRenderCachePipelinedFlush */
5212 I915_WRITE(CACHE_MODE_0,
5213 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
5214
5215 g4x_disable_trickle_feed(dev);
5216}
5217
5218static void crestline_init_clock_gating(struct drm_device *dev)
5219{
5220 struct drm_i915_private *dev_priv = dev->dev_private;
5221
5222 I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
5223 I915_WRITE(RENCLK_GATE_D2, 0);
5224 I915_WRITE(DSPCLK_GATE_D, 0);
5225 I915_WRITE(RAMCLK_GATE_D, 0);
5226 I915_WRITE16(DEUC, 0);
5227 I915_WRITE(MI_ARB_STATE,
5228 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5229}
5230
5231static void broadwater_init_clock_gating(struct drm_device *dev)
5232{
5233 struct drm_i915_private *dev_priv = dev->dev_private;
5234
5235 I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
5236 I965_RCC_CLOCK_GATE_DISABLE |
5237 I965_RCPB_CLOCK_GATE_DISABLE |
5238 I965_ISC_CLOCK_GATE_DISABLE |
5239 I965_FBC_CLOCK_GATE_DISABLE);
5240 I915_WRITE(RENCLK_GATE_D2, 0);
5241 I915_WRITE(MI_ARB_STATE,
5242 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5243}
5244
5245static void gen3_init_clock_gating(struct drm_device *dev)
5246{
5247 struct drm_i915_private *dev_priv = dev->dev_private;
5248 u32 dstate = I915_READ(D_STATE);
5249
5250 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
5251 DSTATE_DOT_CLOCK_GATING;
5252 I915_WRITE(D_STATE, dstate);
5253
5254 if (IS_PINEVIEW(dev))
5255 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
5256
5257 /* IIR "flip pending" means done if this bit is set */
5258 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5259}
5260
5261static void i85x_init_clock_gating(struct drm_device *dev)
5262{
5263 struct drm_i915_private *dev_priv = dev->dev_private;
5264
5265 I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5266}
5267
5268static void i830_init_clock_gating(struct drm_device *dev)
5269{
5270 struct drm_i915_private *dev_priv = dev->dev_private;
5271
5272 I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5273}
5274
5275void intel_init_clock_gating(struct drm_device *dev)
5276{
5277 struct drm_i915_private *dev_priv = dev->dev_private;
5278
5279 dev_priv->display.init_clock_gating(dev);
5280}
5281
5282void intel_suspend_hw(struct drm_device *dev)
5283{
5284 if (HAS_PCH_LPT(dev))
5285 lpt_suspend_hw(dev);
5286}
5287
5288#define for_each_power_well(i, power_well, domain_mask, power_domains) \
5289 for (i = 0; \
5290 i < (power_domains)->power_well_count && \
5291 ((power_well) = &(power_domains)->power_wells[i]); \
5292 i++) \
5293 if ((power_well)->domains & (domain_mask))
5294
5295#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
5296 for (i = (power_domains)->power_well_count - 1; \
5297 i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
5298 i--) \
5299 if ((power_well)->domains & (domain_mask))
5300
5301/**
5302 * We should only use the power well if we explicitly asked the hardware to
5303 * enable it, so check if it's enabled and also check if we've requested it to
5304 * be enabled.
5305 */
5306static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
5307 struct i915_power_well *power_well)
5308{
5309 return I915_READ(HSW_PWR_WELL_DRIVER) ==
5310 (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
5311}
5312
5313bool intel_display_power_enabled_sw(struct drm_i915_private *dev_priv,
5314 enum intel_display_power_domain domain)
5315{
5316 struct i915_power_domains *power_domains;
5317
5318 power_domains = &dev_priv->power_domains;
5319
5320 return power_domains->domain_use_count[domain];
5321}
5322
5323bool intel_display_power_enabled(struct drm_i915_private *dev_priv,
5324 enum intel_display_power_domain domain)
5325{
5326 struct i915_power_domains *power_domains;
5327 struct i915_power_well *power_well;
5328 bool is_enabled;
5329 int i;
5330
5331 if (dev_priv->pm.suspended)
5332 return false;
5333
5334 power_domains = &dev_priv->power_domains;
5335
5336 is_enabled = true;
5337
5338 mutex_lock(&power_domains->lock);
5339 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
5340 if (power_well->always_on)
5341 continue;
5342
5343 if (!power_well->ops->is_enabled(dev_priv, power_well)) {
5344 is_enabled = false;
5345 break;
5346 }
5347 }
5348 mutex_unlock(&power_domains->lock);
5349
5350 return is_enabled;
5351}
5352
5353/*
5354 * Starting with Haswell, we have a "Power Down Well" that can be turned off
5355 * when not needed anymore. We have 4 registers that can request the power well
5356 * to be enabled, and it will only be disabled if none of the registers is
5357 * requesting it to be enabled.
5358 */
5359static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
5360{
5361 struct drm_device *dev = dev_priv->dev;
5362 unsigned long irqflags;
5363
5364 /*
5365 * After we re-enable the power well, if we touch VGA register 0x3d5
5366 * we'll get unclaimed register interrupts. This stops after we write
5367 * anything to the VGA MSR register. The vgacon module uses this
5368 * register all the time, so if we unbind our driver and, as a
5369 * consequence, bind vgacon, we'll get stuck in an infinite loop at
5370 * console_unlock(). So make here we touch the VGA MSR register, making
5371 * sure vgacon can keep working normally without triggering interrupts
5372 * and error messages.
5373 */
5374 vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
5375 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
5376 vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
5377
5378 if (IS_BROADWELL(dev)) {
5379 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
5380 I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B),
5381 dev_priv->de_irq_mask[PIPE_B]);
5382 I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B),
5383 ~dev_priv->de_irq_mask[PIPE_B] |
5384 GEN8_PIPE_VBLANK);
5385 I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C),
5386 dev_priv->de_irq_mask[PIPE_C]);
5387 I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C),
5388 ~dev_priv->de_irq_mask[PIPE_C] |
5389 GEN8_PIPE_VBLANK);
5390 POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C));
5391 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
5392 }
5393}
5394
5395static void reset_vblank_counter(struct drm_device *dev, enum pipe pipe)
5396{
5397 assert_spin_locked(&dev->vbl_lock);
5398
5399 dev->vblank[pipe].last = 0;
5400}
5401
5402static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv)
5403{
5404 struct drm_device *dev = dev_priv->dev;
5405 enum pipe pipe;
5406 unsigned long irqflags;
5407
5408 /*
5409 * After this, the registers on the pipes that are part of the power
5410 * well will become zero, so we have to adjust our counters according to
5411 * that.
5412 *
5413 * FIXME: Should we do this in general in drm_vblank_post_modeset?
5414 */
5415 spin_lock_irqsave(&dev->vbl_lock, irqflags);
5416 for_each_pipe(pipe)
5417 if (pipe != PIPE_A)
5418 reset_vblank_counter(dev, pipe);
5419 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
5420}
5421
5422static void hsw_set_power_well(struct drm_i915_private *dev_priv,
5423 struct i915_power_well *power_well, bool enable)
5424{
5425 bool is_enabled, enable_requested;
5426 uint32_t tmp;
5427
5428 tmp = I915_READ(HSW_PWR_WELL_DRIVER);
5429 is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
5430 enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
5431
5432 if (enable) {
5433 if (!enable_requested)
5434 I915_WRITE(HSW_PWR_WELL_DRIVER,
5435 HSW_PWR_WELL_ENABLE_REQUEST);
5436
5437 if (!is_enabled) {
5438 DRM_DEBUG_KMS("Enabling power well\n");
5439 if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
5440 HSW_PWR_WELL_STATE_ENABLED), 20))
5441 DRM_ERROR("Timeout enabling power well\n");
5442 }
5443
5444 hsw_power_well_post_enable(dev_priv);
5445 } else {
5446 if (enable_requested) {
5447 I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
5448 POSTING_READ(HSW_PWR_WELL_DRIVER);
5449 DRM_DEBUG_KMS("Requesting to disable the power well\n");
5450
5451 hsw_power_well_post_disable(dev_priv);
5452 }
5453 }
5454}
5455
5456static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
5457 struct i915_power_well *power_well)
5458{
5459 hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
5460
5461 /*
5462 * We're taking over the BIOS, so clear any requests made by it since
5463 * the driver is in charge now.
5464 */
5465 if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
5466 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
5467}
5468
5469static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
5470 struct i915_power_well *power_well)
5471{
5472 hsw_set_power_well(dev_priv, power_well, true);
5473}
5474
5475static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
5476 struct i915_power_well *power_well)
5477{
5478 hsw_set_power_well(dev_priv, power_well, false);
5479}
5480
5481static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
5482 struct i915_power_well *power_well)
5483{
5484}
5485
5486static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
5487 struct i915_power_well *power_well)
5488{
5489 return true;
5490}
5491
5492static void vlv_set_power_well(struct drm_i915_private *dev_priv,
5493 struct i915_power_well *power_well, bool enable)
5494{
5495 enum punit_power_well power_well_id = power_well->data;
5496 u32 mask;
5497 u32 state;
5498 u32 ctrl;
5499
5500 mask = PUNIT_PWRGT_MASK(power_well_id);
5501 state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
5502 PUNIT_PWRGT_PWR_GATE(power_well_id);
5503
5504 mutex_lock(&dev_priv->rps.hw_lock);
5505
5506#define COND \
5507 ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
5508
5509 if (COND)
5510 goto out;
5511
5512 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
5513 ctrl &= ~mask;
5514 ctrl |= state;
5515 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
5516
5517 if (wait_for(COND, 100))
5518 DRM_ERROR("timout setting power well state %08x (%08x)\n",
5519 state,
5520 vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
5521
5522#undef COND
5523
5524out:
5525 mutex_unlock(&dev_priv->rps.hw_lock);
5526}
5527
5528static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
5529 struct i915_power_well *power_well)
5530{
5531 vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
5532}
5533
5534static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
5535 struct i915_power_well *power_well)
5536{
5537 vlv_set_power_well(dev_priv, power_well, true);
5538}
5539
5540static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
5541 struct i915_power_well *power_well)
5542{
5543 vlv_set_power_well(dev_priv, power_well, false);
5544}
5545
5546static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
5547 struct i915_power_well *power_well)
5548{
5549 int power_well_id = power_well->data;
5550 bool enabled = false;
5551 u32 mask;
5552 u32 state;
5553 u32 ctrl;
5554
5555 mask = PUNIT_PWRGT_MASK(power_well_id);
5556 ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
5557
5558 mutex_lock(&dev_priv->rps.hw_lock);
5559
5560 state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
5561 /*
5562 * We only ever set the power-on and power-gate states, anything
5563 * else is unexpected.
5564 */
5565 WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
5566 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
5567 if (state == ctrl)
5568 enabled = true;
5569
5570 /*
5571 * A transient state at this point would mean some unexpected party
5572 * is poking at the power controls too.
5573 */
5574 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
5575 WARN_ON(ctrl != state);
5576
5577 mutex_unlock(&dev_priv->rps.hw_lock);
5578
5579 return enabled;
5580}
5581
5582static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
5583 struct i915_power_well *power_well)
5584{
5585 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
5586
5587 vlv_set_power_well(dev_priv, power_well, true);
5588
5589 spin_lock_irq(&dev_priv->irq_lock);
5590 valleyview_enable_display_irqs(dev_priv);
5591 spin_unlock_irq(&dev_priv->irq_lock);
5592
5593 /*
5594 * During driver initialization we need to defer enabling hotplug
5595 * processing until fbdev is set up.
5596 */
5597 if (dev_priv->enable_hotplug_processing)
5598 intel_hpd_init(dev_priv->dev);
5599
5600 i915_redisable_vga_power_on(dev_priv->dev);
5601}
5602
5603static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
5604 struct i915_power_well *power_well)
5605{
5606 struct drm_device *dev = dev_priv->dev;
5607 enum pipe pipe;
5608
5609 WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
5610
5611 spin_lock_irq(&dev_priv->irq_lock);
5612 for_each_pipe(pipe)
5613 __intel_set_cpu_fifo_underrun_reporting(dev, pipe, false);
5614
5615 valleyview_disable_display_irqs(dev_priv);
5616 spin_unlock_irq(&dev_priv->irq_lock);
5617
5618 spin_lock_irq(&dev->vbl_lock);
5619 for_each_pipe(pipe)
5620 reset_vblank_counter(dev, pipe);
5621 spin_unlock_irq(&dev->vbl_lock);
5622
5623 vlv_set_power_well(dev_priv, power_well, false);
5624}
5625
5626static void check_power_well_state(struct drm_i915_private *dev_priv,
5627 struct i915_power_well *power_well)
5628{
5629 bool enabled = power_well->ops->is_enabled(dev_priv, power_well);
5630
5631 if (power_well->always_on || !i915.disable_power_well) {
5632 if (!enabled)
5633 goto mismatch;
5634
5635 return;
5636 }
5637
5638 if (enabled != (power_well->count > 0))
5639 goto mismatch;
5640
5641 return;
5642
5643mismatch:
5644 WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n",
5645 power_well->name, power_well->always_on, enabled,
5646 power_well->count, i915.disable_power_well);
5647}
5648
5649void intel_display_power_get(struct drm_i915_private *dev_priv,
5650 enum intel_display_power_domain domain)
5651{
5652 struct i915_power_domains *power_domains;
5653 struct i915_power_well *power_well;
5654 int i;
5655
5656 intel_runtime_pm_get(dev_priv);
5657
5658 power_domains = &dev_priv->power_domains;
5659
5660 mutex_lock(&power_domains->lock);
5661
5662 for_each_power_well(i, power_well, BIT(domain), power_domains) {
5663 if (!power_well->count++) {
5664 DRM_DEBUG_KMS("enabling %s\n", power_well->name);
5665 power_well->ops->enable(dev_priv, power_well);
5666 }
5667
5668 check_power_well_state(dev_priv, power_well);
5669 }
5670
5671 power_domains->domain_use_count[domain]++;
5672
5673 mutex_unlock(&power_domains->lock);
5674}
5675
5676void intel_display_power_put(struct drm_i915_private *dev_priv,
5677 enum intel_display_power_domain domain)
5678{
5679 struct i915_power_domains *power_domains;
5680 struct i915_power_well *power_well;
5681 int i;
5682
5683 power_domains = &dev_priv->power_domains;
5684
5685 mutex_lock(&power_domains->lock);
5686
5687 WARN_ON(!power_domains->domain_use_count[domain]);
5688 power_domains->domain_use_count[domain]--;
5689
5690 for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
5691 WARN_ON(!power_well->count);
5692
5693 if (!--power_well->count && i915.disable_power_well) {
5694 DRM_DEBUG_KMS("disabling %s\n", power_well->name);
5695 power_well->ops->disable(dev_priv, power_well);
5696 }
5697
5698 check_power_well_state(dev_priv, power_well);
5699 }
5700
5701 mutex_unlock(&power_domains->lock);
5702
5703 intel_runtime_pm_put(dev_priv);
5704}
5705
5706static struct i915_power_domains *hsw_pwr;
5707
5708/* Display audio driver power well request */
5709void i915_request_power_well(void)
5710{
5711 struct drm_i915_private *dev_priv;
5712
5713 if (WARN_ON(!hsw_pwr))
5714 return;
5715
5716 dev_priv = container_of(hsw_pwr, struct drm_i915_private,
5717 power_domains);
5718 intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
5719}
5720EXPORT_SYMBOL_GPL(i915_request_power_well);
5721
5722/* Display audio driver power well release */
5723void i915_release_power_well(void)
5724{
5725 struct drm_i915_private *dev_priv;
5726
5727 if (WARN_ON(!hsw_pwr))
5728 return;
5729
5730 dev_priv = container_of(hsw_pwr, struct drm_i915_private,
5731 power_domains);
5732 intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
5733}
5734EXPORT_SYMBOL_GPL(i915_release_power_well);
5735
5736#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
5737
5738#define HSW_ALWAYS_ON_POWER_DOMAINS ( \
5739 BIT(POWER_DOMAIN_PIPE_A) | \
5740 BIT(POWER_DOMAIN_TRANSCODER_EDP) | \
5741 BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \
5742 BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \
5743 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
5744 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
5745 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
5746 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
5747 BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \
5748 BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \
5749 BIT(POWER_DOMAIN_PORT_CRT) | \
5750 BIT(POWER_DOMAIN_INIT))
5751#define HSW_DISPLAY_POWER_DOMAINS ( \
5752 (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \
5753 BIT(POWER_DOMAIN_INIT))
5754
5755#define BDW_ALWAYS_ON_POWER_DOMAINS ( \
5756 HSW_ALWAYS_ON_POWER_DOMAINS | \
5757 BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
5758#define BDW_DISPLAY_POWER_DOMAINS ( \
5759 (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \
5760 BIT(POWER_DOMAIN_INIT))
5761
5762#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT)
5763#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK
5764
5765#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \
5766 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
5767 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
5768 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
5769 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
5770 BIT(POWER_DOMAIN_PORT_CRT) | \
5771 BIT(POWER_DOMAIN_INIT))
5772
5773#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \
5774 BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \
5775 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
5776 BIT(POWER_DOMAIN_INIT))
5777
5778#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \
5779 BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \
5780 BIT(POWER_DOMAIN_INIT))
5781
5782#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \
5783 BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \
5784 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
5785 BIT(POWER_DOMAIN_INIT))
5786
5787#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \
5788 BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \
5789 BIT(POWER_DOMAIN_INIT))
5790
5791static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
5792 .sync_hw = i9xx_always_on_power_well_noop,
5793 .enable = i9xx_always_on_power_well_noop,
5794 .disable = i9xx_always_on_power_well_noop,
5795 .is_enabled = i9xx_always_on_power_well_enabled,
5796};
5797
5798static struct i915_power_well i9xx_always_on_power_well[] = {
5799 {
5800 .name = "always-on",
5801 .always_on = 1,
5802 .domains = POWER_DOMAIN_MASK,
5803 .ops = &i9xx_always_on_power_well_ops,
5804 },
5805};
5806
5807static const struct i915_power_well_ops hsw_power_well_ops = {
5808 .sync_hw = hsw_power_well_sync_hw,
5809 .enable = hsw_power_well_enable,
5810 .disable = hsw_power_well_disable,
5811 .is_enabled = hsw_power_well_enabled,
5812};
5813
5814static struct i915_power_well hsw_power_wells[] = {
5815 {
5816 .name = "always-on",
5817 .always_on = 1,
5818 .domains = HSW_ALWAYS_ON_POWER_DOMAINS,
5819 .ops = &i9xx_always_on_power_well_ops,
5820 },
5821 {
5822 .name = "display",
5823 .domains = HSW_DISPLAY_POWER_DOMAINS,
5824 .ops = &hsw_power_well_ops,
5825 },
5826};
5827
5828static struct i915_power_well bdw_power_wells[] = {
5829 {
5830 .name = "always-on",
5831 .always_on = 1,
5832 .domains = BDW_ALWAYS_ON_POWER_DOMAINS,
5833 .ops = &i9xx_always_on_power_well_ops,
5834 },
5835 {
5836 .name = "display",
5837 .domains = BDW_DISPLAY_POWER_DOMAINS,
5838 .ops = &hsw_power_well_ops,
5839 },
5840};
5841
5842static const struct i915_power_well_ops vlv_display_power_well_ops = {
5843 .sync_hw = vlv_power_well_sync_hw,
5844 .enable = vlv_display_power_well_enable,
5845 .disable = vlv_display_power_well_disable,
5846 .is_enabled = vlv_power_well_enabled,
5847};
5848
5849static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
5850 .sync_hw = vlv_power_well_sync_hw,
5851 .enable = vlv_power_well_enable,
5852 .disable = vlv_power_well_disable,
5853 .is_enabled = vlv_power_well_enabled,
5854};
5855
5856static struct i915_power_well vlv_power_wells[] = {
5857 {
5858 .name = "always-on",
5859 .always_on = 1,
5860 .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
5861 .ops = &i9xx_always_on_power_well_ops,
5862 },
5863 {
5864 .name = "display",
5865 .domains = VLV_DISPLAY_POWER_DOMAINS,
5866 .data = PUNIT_POWER_WELL_DISP2D,
5867 .ops = &vlv_display_power_well_ops,
5868 },
5869 {
5870 .name = "dpio-common",
5871 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
5872 .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
5873 .ops = &vlv_dpio_power_well_ops,
5874 },
5875 {
5876 .name = "dpio-tx-b-01",
5877 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
5878 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
5879 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
5880 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
5881 .ops = &vlv_dpio_power_well_ops,
5882 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
5883 },
5884 {
5885 .name = "dpio-tx-b-23",
5886 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
5887 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
5888 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
5889 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
5890 .ops = &vlv_dpio_power_well_ops,
5891 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
5892 },
5893 {
5894 .name = "dpio-tx-c-01",
5895 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
5896 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
5897 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
5898 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
5899 .ops = &vlv_dpio_power_well_ops,
5900 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
5901 },
5902 {
5903 .name = "dpio-tx-c-23",
5904 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
5905 VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
5906 VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
5907 VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
5908 .ops = &vlv_dpio_power_well_ops,
5909 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
5910 },
5911};
5912
5913#define set_power_wells(power_domains, __power_wells) ({ \
5914 (power_domains)->power_wells = (__power_wells); \
5915 (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \
5916})
5917
5918int intel_power_domains_init(struct drm_i915_private *dev_priv)
5919{
5920 struct i915_power_domains *power_domains = &dev_priv->power_domains;
5921
5922 mutex_init(&power_domains->lock);
5923
5924 /*
5925 * The enabling order will be from lower to higher indexed wells,
5926 * the disabling order is reversed.
5927 */
5928 if (IS_HASWELL(dev_priv->dev)) {
5929 set_power_wells(power_domains, hsw_power_wells);
5930 hsw_pwr = power_domains;
5931 } else if (IS_BROADWELL(dev_priv->dev)) {
5932 set_power_wells(power_domains, bdw_power_wells);
5933 hsw_pwr = power_domains;
5934 } else if (IS_VALLEYVIEW(dev_priv->dev)) {
5935 set_power_wells(power_domains, vlv_power_wells);
5936 } else {
5937 set_power_wells(power_domains, i9xx_always_on_power_well);
5938 }
5939
5940 return 0;
5941}
5942
5943void intel_power_domains_remove(struct drm_i915_private *dev_priv)
5944{
5945 hsw_pwr = NULL;
5946}
5947
5948static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
5949{
5950 struct i915_power_domains *power_domains = &dev_priv->power_domains;
5951 struct i915_power_well *power_well;
5952 int i;
5953
5954 mutex_lock(&power_domains->lock);
5955 for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains)
5956 power_well->ops->sync_hw(dev_priv, power_well);
5957 mutex_unlock(&power_domains->lock);
5958}
5959
5960void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
5961{
5962 /* For now, we need the power well to be always enabled. */
5963 intel_display_set_init_power(dev_priv, true);
5964 intel_power_domains_resume(dev_priv);
5965}
5966
5967void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
5968{
5969 intel_runtime_pm_get(dev_priv);
5970}
5971
5972void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
5973{
5974 intel_runtime_pm_put(dev_priv);
5975}
5976
5977void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
5978{
5979 struct drm_device *dev = dev_priv->dev;
5980 struct device *device = &dev->pdev->dev;
5981
5982 if (!HAS_RUNTIME_PM(dev))
5983 return;
5984
5985 pm_runtime_get_sync(device);
5986 WARN(dev_priv->pm.suspended, "Device still suspended.\n");
5987}
5988
5989void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
5990{
5991 struct drm_device *dev = dev_priv->dev;
5992 struct device *device = &dev->pdev->dev;
5993
5994 if (!HAS_RUNTIME_PM(dev))
5995 return;
5996
5997 pm_runtime_mark_last_busy(device);
5998 pm_runtime_put_autosuspend(device);
5999}
6000
6001void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
6002{
6003 struct drm_device *dev = dev_priv->dev;
6004 struct device *device = &dev->pdev->dev;
6005
6006 if (!HAS_RUNTIME_PM(dev))
6007 return;
6008
6009 pm_runtime_set_active(device);
6010
6011 pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
6012 pm_runtime_mark_last_busy(device);
6013 pm_runtime_use_autosuspend(device);
6014
6015 pm_runtime_put_autosuspend(device);
6016}
6017
6018void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
6019{
6020 struct drm_device *dev = dev_priv->dev;
6021 struct device *device = &dev->pdev->dev;
6022
6023 if (!HAS_RUNTIME_PM(dev))
6024 return;
6025
6026 /* Make sure we're not suspended first. */
6027 pm_runtime_get_sync(device);
6028 pm_runtime_disable(device);
6029}
6030
6031/* Set up chip specific power management-related functions */
6032void intel_init_pm(struct drm_device *dev)
6033{
6034 struct drm_i915_private *dev_priv = dev->dev_private;
6035
6036 if (HAS_FBC(dev)) {
6037 if (INTEL_INFO(dev)->gen >= 7) {
6038 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
6039 dev_priv->display.enable_fbc = gen7_enable_fbc;
6040 dev_priv->display.disable_fbc = ironlake_disable_fbc;
6041 } else if (INTEL_INFO(dev)->gen >= 5) {
6042 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
6043 dev_priv->display.enable_fbc = ironlake_enable_fbc;
6044 dev_priv->display.disable_fbc = ironlake_disable_fbc;
6045 } else if (IS_GM45(dev)) {
6046 dev_priv->display.fbc_enabled = g4x_fbc_enabled;
6047 dev_priv->display.enable_fbc = g4x_enable_fbc;
6048 dev_priv->display.disable_fbc = g4x_disable_fbc;
6049 } else {
6050 dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
6051 dev_priv->display.enable_fbc = i8xx_enable_fbc;
6052 dev_priv->display.disable_fbc = i8xx_disable_fbc;
6053
6054 /* This value was pulled out of someone's hat */
6055 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
6056 }
6057 }
6058
6059 /* For cxsr */
6060 if (IS_PINEVIEW(dev))
6061 i915_pineview_get_mem_freq(dev);
6062 else if (IS_GEN5(dev))
6063 i915_ironlake_get_mem_freq(dev);
6064
6065 /* For FIFO watermark updates */
6066 if (HAS_PCH_SPLIT(dev)) {
6067 ilk_setup_wm_latency(dev);
6068
6069 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
6070 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
6071 (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
6072 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
6073 dev_priv->display.update_wm = ilk_update_wm;
6074 dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6075 } else {
6076 DRM_DEBUG_KMS("Failed to read display plane latency. "
6077 "Disable CxSR\n");
6078 }
6079
6080 if (IS_GEN5(dev))
6081 dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
6082 else if (IS_GEN6(dev))
6083 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
6084 else if (IS_IVYBRIDGE(dev))
6085 dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
6086 else if (IS_HASWELL(dev))
6087 dev_priv->display.init_clock_gating = haswell_init_clock_gating;
6088 else if (INTEL_INFO(dev)->gen == 8)
6089 dev_priv->display.init_clock_gating = gen8_init_clock_gating;
6090 } else if (IS_VALLEYVIEW(dev)) {
6091 dev_priv->display.update_wm = valleyview_update_wm;
6092 dev_priv->display.init_clock_gating =
6093 valleyview_init_clock_gating;
6094 } else if (IS_PINEVIEW(dev)) {
6095 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
6096 dev_priv->is_ddr3,
6097 dev_priv->fsb_freq,
6098 dev_priv->mem_freq)) {
6099 DRM_INFO("failed to find known CxSR latency "
6100 "(found ddr%s fsb freq %d, mem freq %d), "
6101 "disabling CxSR\n",
6102 (dev_priv->is_ddr3 == 1) ? "3" : "2",
6103 dev_priv->fsb_freq, dev_priv->mem_freq);
6104 /* Disable CxSR and never update its watermark again */
6105 pineview_disable_cxsr(dev);
6106 dev_priv->display.update_wm = NULL;
6107 } else
6108 dev_priv->display.update_wm = pineview_update_wm;
6109 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6110 } else if (IS_G4X(dev)) {
6111 dev_priv->display.update_wm = g4x_update_wm;
6112 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
6113 } else if (IS_GEN4(dev)) {
6114 dev_priv->display.update_wm = i965_update_wm;
6115 if (IS_CRESTLINE(dev))
6116 dev_priv->display.init_clock_gating = crestline_init_clock_gating;
6117 else if (IS_BROADWATER(dev))
6118 dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
6119 } else if (IS_GEN3(dev)) {
6120 dev_priv->display.update_wm = i9xx_update_wm;
6121 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
6122 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6123 } else if (IS_GEN2(dev)) {
6124 if (INTEL_INFO(dev)->num_pipes == 1) {
6125 dev_priv->display.update_wm = i845_update_wm;
6126 dev_priv->display.get_fifo_size = i845_get_fifo_size;
6127 } else {
6128 dev_priv->display.update_wm = i9xx_update_wm;
6129 dev_priv->display.get_fifo_size = i830_get_fifo_size;
6130 }
6131
6132 if (IS_I85X(dev) || IS_I865G(dev))
6133 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
6134 else
6135 dev_priv->display.init_clock_gating = i830_init_clock_gating;
6136 } else {
6137 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
6138 }
6139}
6140
6141int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
6142{
6143 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6144
6145 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6146 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6147 return -EAGAIN;
6148 }
6149
6150 I915_WRITE(GEN6_PCODE_DATA, *val);
6151 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6152
6153 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6154 500)) {
6155 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6156 return -ETIMEDOUT;
6157 }
6158
6159 *val = I915_READ(GEN6_PCODE_DATA);
6160 I915_WRITE(GEN6_PCODE_DATA, 0);
6161
6162 return 0;
6163}
6164
6165int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
6166{
6167 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6168
6169 if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6170 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6171 return -EAGAIN;
6172 }
6173
6174 I915_WRITE(GEN6_PCODE_DATA, val);
6175 I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6176
6177 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6178 500)) {
6179 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6180 return -ETIMEDOUT;
6181 }
6182
6183 I915_WRITE(GEN6_PCODE_DATA, 0);
6184
6185 return 0;
6186}
6187
6188int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
6189{
6190 int div;
6191
6192 /* 4 x czclk */
6193 switch (dev_priv->mem_freq) {
6194 case 800:
6195 div = 10;
6196 break;
6197 case 1066:
6198 div = 12;
6199 break;
6200 case 1333:
6201 div = 16;
6202 break;
6203 default:
6204 return -1;
6205 }
6206
6207 return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div);
6208}
6209
6210int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
6211{
6212 int mul;
6213
6214 /* 4 x czclk */
6215 switch (dev_priv->mem_freq) {
6216 case 800:
6217 mul = 10;
6218 break;
6219 case 1066:
6220 mul = 12;
6221 break;
6222 case 1333:
6223 mul = 16;
6224 break;
6225 default:
6226 return -1;
6227 }
6228
6229 return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6;
6230}
6231
6232void intel_pm_setup(struct drm_device *dev)
6233{
6234 struct drm_i915_private *dev_priv = dev->dev_private;
6235
6236 mutex_init(&dev_priv->rps.hw_lock);
6237
6238 INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6239 intel_gen6_powersave_work);
6240
6241 dev_priv->pm.suspended = false;
6242 dev_priv->pm.irqs_disabled = false;
6243}