Loading...
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "i915_reg.h"
8#include "i9xx_wm.h"
9#include "intel_atomic.h"
10#include "intel_bo.h"
11#include "intel_display.h"
12#include "intel_display_trace.h"
13#include "intel_fb.h"
14#include "intel_mchbar_regs.h"
15#include "intel_wm.h"
16#include "skl_watermark.h"
17#include "vlv_sideband.h"
18
19struct intel_watermark_params {
20 u16 fifo_size;
21 u16 max_wm;
22 u8 default_wm;
23 u8 guard_size;
24 u8 cacheline_size;
25};
26
27/* used in computing the new watermarks state */
28struct intel_wm_config {
29 unsigned int num_pipes_active;
30 bool sprites_enabled;
31 bool sprites_scaled;
32};
33
34struct cxsr_latency {
35 bool is_desktop : 1;
36 bool is_ddr3 : 1;
37 u16 fsb_freq;
38 u16 mem_freq;
39 u16 display_sr;
40 u16 display_hpll_disable;
41 u16 cursor_sr;
42 u16 cursor_hpll_disable;
43};
44
45static const struct cxsr_latency cxsr_latency_table[] = {
46 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
47 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
48 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
49 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
50 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
51
52 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
53 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
54 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
55 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
56 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
57
58 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
59 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
60 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
61 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
62 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
63
64 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
65 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
66 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
67 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
68 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
69
70 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
71 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
72 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
73 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
74 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
75
76 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
77 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
78 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
79 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
80 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
81};
82
83static const struct cxsr_latency *pnv_get_cxsr_latency(struct drm_i915_private *i915)
84{
85 int i;
86
87 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
88 const struct cxsr_latency *latency = &cxsr_latency_table[i];
89 bool is_desktop = !IS_MOBILE(i915);
90
91 if (is_desktop == latency->is_desktop &&
92 i915->is_ddr3 == latency->is_ddr3 &&
93 DIV_ROUND_CLOSEST(i915->fsb_freq, 1000) == latency->fsb_freq &&
94 DIV_ROUND_CLOSEST(i915->mem_freq, 1000) == latency->mem_freq)
95 return latency;
96 }
97
98 drm_dbg_kms(&i915->drm,
99 "Could not find CxSR latency for DDR%s, FSB %u kHz, MEM %u kHz\n",
100 i915->is_ddr3 ? "3" : "2", i915->fsb_freq, i915->mem_freq);
101
102 return NULL;
103}
104
105static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
106{
107 u32 val;
108
109 vlv_punit_get(dev_priv);
110
111 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
112 if (enable)
113 val &= ~FORCE_DDR_HIGH_FREQ;
114 else
115 val |= FORCE_DDR_HIGH_FREQ;
116 val &= ~FORCE_DDR_LOW_FREQ;
117 val |= FORCE_DDR_FREQ_REQ_ACK;
118 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
119
120 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
121 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
122 drm_err(&dev_priv->drm,
123 "timed out waiting for Punit DDR DVFS request\n");
124
125 vlv_punit_put(dev_priv);
126}
127
128static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
129{
130 u32 val;
131
132 vlv_punit_get(dev_priv);
133
134 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
135 if (enable)
136 val |= DSP_MAXFIFO_PM5_ENABLE;
137 else
138 val &= ~DSP_MAXFIFO_PM5_ENABLE;
139 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
140
141 vlv_punit_put(dev_priv);
142}
143
144#define FW_WM(value, plane) \
145 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
146
147static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
148{
149 struct intel_display *display = &dev_priv->display;
150 bool was_enabled;
151 u32 val;
152
153 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
154 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
155 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
156 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
157 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
158 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
159 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
160 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
161 } else if (IS_PINEVIEW(dev_priv)) {
162 val = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
163 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
164 if (enable)
165 val |= PINEVIEW_SELF_REFRESH_EN;
166 else
167 val &= ~PINEVIEW_SELF_REFRESH_EN;
168 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), val);
169 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3(dev_priv));
170 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
171 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
172 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
173 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
174 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
175 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
176 } else if (IS_I915GM(dev_priv)) {
177 /*
178 * FIXME can't find a bit like this for 915G, and
179 * yet it does have the related watermark in
180 * FW_BLC_SELF. What's going on?
181 */
182 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
183 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
184 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
185 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
186 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
187 } else {
188 return false;
189 }
190
191 trace_intel_memory_cxsr(display, was_enabled, enable);
192
193 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
194 str_enabled_disabled(enable),
195 str_enabled_disabled(was_enabled));
196
197 return was_enabled;
198}
199
200/**
201 * intel_set_memory_cxsr - Configure CxSR state
202 * @dev_priv: i915 device
203 * @enable: Allow vs. disallow CxSR
204 *
205 * Allow or disallow the system to enter a special CxSR
206 * (C-state self refresh) state. What typically happens in CxSR mode
207 * is that several display FIFOs may get combined into a single larger
208 * FIFO for a particular plane (so called max FIFO mode) to allow the
209 * system to defer memory fetches longer, and the memory will enter
210 * self refresh.
211 *
212 * Note that enabling CxSR does not guarantee that the system enter
213 * this special mode, nor does it guarantee that the system stays
214 * in that mode once entered. So this just allows/disallows the system
215 * to autonomously utilize the CxSR mode. Other factors such as core
216 * C-states will affect when/if the system actually enters/exits the
217 * CxSR mode.
218 *
219 * Note that on VLV/CHV this actually only controls the max FIFO mode,
220 * and the system is free to enter/exit memory self refresh at any time
221 * even when the use of CxSR has been disallowed.
222 *
223 * While the system is actually in the CxSR/max FIFO mode, some plane
224 * control registers will not get latched on vblank. Thus in order to
225 * guarantee the system will respond to changes in the plane registers
226 * we must always disallow CxSR prior to making changes to those registers.
227 * Unfortunately the system will re-evaluate the CxSR conditions at
228 * frame start which happens after vblank start (which is when the plane
229 * registers would get latched), so we can't proceed with the plane update
230 * during the same frame where we disallowed CxSR.
231 *
232 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
233 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
234 * the hardware w.r.t. HPLL SR when writing to plane registers.
235 * Disallowing just CxSR is sufficient.
236 */
237bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
238{
239 bool ret;
240
241 mutex_lock(&dev_priv->display.wm.wm_mutex);
242 ret = _intel_set_memory_cxsr(dev_priv, enable);
243 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
244 dev_priv->display.wm.vlv.cxsr = enable;
245 else if (IS_G4X(dev_priv))
246 dev_priv->display.wm.g4x.cxsr = enable;
247 mutex_unlock(&dev_priv->display.wm.wm_mutex);
248
249 return ret;
250}
251
252/*
253 * Latency for FIFO fetches is dependent on several factors:
254 * - memory configuration (speed, channels)
255 * - chipset
256 * - current MCH state
257 * It can be fairly high in some situations, so here we assume a fairly
258 * pessimal value. It's a tradeoff between extra memory fetches (if we
259 * set this value too high, the FIFO will fetch frequently to stay full)
260 * and power consumption (set it too low to save power and we might see
261 * FIFO underruns and display "flicker").
262 *
263 * A value of 5us seems to be a good balance; safe for very low end
264 * platforms but not overly aggressive on lower latency configs.
265 */
266static const int pessimal_latency_ns = 5000;
267
268#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
269 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
270
271static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
272{
273 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
274 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
275 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
276 enum pipe pipe = crtc->pipe;
277 int sprite0_start, sprite1_start;
278 u32 dsparb, dsparb2, dsparb3;
279
280 switch (pipe) {
281 case PIPE_A:
282 dsparb = intel_uncore_read(&dev_priv->uncore,
283 DSPARB(dev_priv));
284 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
285 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
286 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
287 break;
288 case PIPE_B:
289 dsparb = intel_uncore_read(&dev_priv->uncore,
290 DSPARB(dev_priv));
291 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
292 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
293 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
294 break;
295 case PIPE_C:
296 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
297 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
298 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
299 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
300 break;
301 default:
302 MISSING_CASE(pipe);
303 return;
304 }
305
306 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
307 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
308 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
309 fifo_state->plane[PLANE_CURSOR] = 63;
310}
311
312static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
313 enum i9xx_plane_id i9xx_plane)
314{
315 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
316 int size;
317
318 size = dsparb & 0x7f;
319 if (i9xx_plane == PLANE_B)
320 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
321
322 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
323 dsparb, plane_name(i9xx_plane), size);
324
325 return size;
326}
327
328static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
329 enum i9xx_plane_id i9xx_plane)
330{
331 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
332 int size;
333
334 size = dsparb & 0x1ff;
335 if (i9xx_plane == PLANE_B)
336 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
337 size >>= 1; /* Convert to cachelines */
338
339 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
340 dsparb, plane_name(i9xx_plane), size);
341
342 return size;
343}
344
345static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
346 enum i9xx_plane_id i9xx_plane)
347{
348 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
349 int size;
350
351 size = dsparb & 0x7f;
352 size >>= 2; /* Convert to cachelines */
353
354 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
355 dsparb, plane_name(i9xx_plane), size);
356
357 return size;
358}
359
360/* Pineview has different values for various configs */
361static const struct intel_watermark_params pnv_display_wm = {
362 .fifo_size = PINEVIEW_DISPLAY_FIFO,
363 .max_wm = PINEVIEW_MAX_WM,
364 .default_wm = PINEVIEW_DFT_WM,
365 .guard_size = PINEVIEW_GUARD_WM,
366 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
367};
368
369static const struct intel_watermark_params pnv_display_hplloff_wm = {
370 .fifo_size = PINEVIEW_DISPLAY_FIFO,
371 .max_wm = PINEVIEW_MAX_WM,
372 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
373 .guard_size = PINEVIEW_GUARD_WM,
374 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
375};
376
377static const struct intel_watermark_params pnv_cursor_wm = {
378 .fifo_size = PINEVIEW_CURSOR_FIFO,
379 .max_wm = PINEVIEW_CURSOR_MAX_WM,
380 .default_wm = PINEVIEW_CURSOR_DFT_WM,
381 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
382 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
383};
384
385static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
386 .fifo_size = PINEVIEW_CURSOR_FIFO,
387 .max_wm = PINEVIEW_CURSOR_MAX_WM,
388 .default_wm = PINEVIEW_CURSOR_DFT_WM,
389 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
390 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
391};
392
393static const struct intel_watermark_params i965_cursor_wm_info = {
394 .fifo_size = I965_CURSOR_FIFO,
395 .max_wm = I965_CURSOR_MAX_WM,
396 .default_wm = I965_CURSOR_DFT_WM,
397 .guard_size = 2,
398 .cacheline_size = I915_FIFO_LINE_SIZE,
399};
400
401static const struct intel_watermark_params i945_wm_info = {
402 .fifo_size = I945_FIFO_SIZE,
403 .max_wm = I915_MAX_WM,
404 .default_wm = 1,
405 .guard_size = 2,
406 .cacheline_size = I915_FIFO_LINE_SIZE,
407};
408
409static const struct intel_watermark_params i915_wm_info = {
410 .fifo_size = I915_FIFO_SIZE,
411 .max_wm = I915_MAX_WM,
412 .default_wm = 1,
413 .guard_size = 2,
414 .cacheline_size = I915_FIFO_LINE_SIZE,
415};
416
417static const struct intel_watermark_params i830_a_wm_info = {
418 .fifo_size = I855GM_FIFO_SIZE,
419 .max_wm = I915_MAX_WM,
420 .default_wm = 1,
421 .guard_size = 2,
422 .cacheline_size = I830_FIFO_LINE_SIZE,
423};
424
425static const struct intel_watermark_params i830_bc_wm_info = {
426 .fifo_size = I855GM_FIFO_SIZE,
427 .max_wm = I915_MAX_WM / 2,
428 .default_wm = 1,
429 .guard_size = 2,
430 .cacheline_size = I830_FIFO_LINE_SIZE,
431};
432
433static const struct intel_watermark_params i845_wm_info = {
434 .fifo_size = I830_FIFO_SIZE,
435 .max_wm = I915_MAX_WM,
436 .default_wm = 1,
437 .guard_size = 2,
438 .cacheline_size = I830_FIFO_LINE_SIZE,
439};
440
441/**
442 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
443 * @pixel_rate: Pipe pixel rate in kHz
444 * @cpp: Plane bytes per pixel
445 * @latency: Memory wakeup latency in 0.1us units
446 *
447 * Compute the watermark using the method 1 or "small buffer"
448 * formula. The caller may additonally add extra cachelines
449 * to account for TLB misses and clock crossings.
450 *
451 * This method is concerned with the short term drain rate
452 * of the FIFO, ie. it does not account for blanking periods
453 * which would effectively reduce the average drain rate across
454 * a longer period. The name "small" refers to the fact the
455 * FIFO is relatively small compared to the amount of data
456 * fetched.
457 *
458 * The FIFO level vs. time graph might look something like:
459 *
460 * |\ |\
461 * | \ | \
462 * __---__---__ (- plane active, _ blanking)
463 * -> time
464 *
465 * or perhaps like this:
466 *
467 * |\|\ |\|\
468 * __----__----__ (- plane active, _ blanking)
469 * -> time
470 *
471 * Returns:
472 * The watermark in bytes
473 */
474static unsigned int intel_wm_method1(unsigned int pixel_rate,
475 unsigned int cpp,
476 unsigned int latency)
477{
478 u64 ret;
479
480 ret = mul_u32_u32(pixel_rate, cpp * latency);
481 ret = DIV_ROUND_UP_ULL(ret, 10000);
482
483 return ret;
484}
485
486/**
487 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
488 * @pixel_rate: Pipe pixel rate in kHz
489 * @htotal: Pipe horizontal total
490 * @width: Plane width in pixels
491 * @cpp: Plane bytes per pixel
492 * @latency: Memory wakeup latency in 0.1us units
493 *
494 * Compute the watermark using the method 2 or "large buffer"
495 * formula. The caller may additonally add extra cachelines
496 * to account for TLB misses and clock crossings.
497 *
498 * This method is concerned with the long term drain rate
499 * of the FIFO, ie. it does account for blanking periods
500 * which effectively reduce the average drain rate across
501 * a longer period. The name "large" refers to the fact the
502 * FIFO is relatively large compared to the amount of data
503 * fetched.
504 *
505 * The FIFO level vs. time graph might look something like:
506 *
507 * |\___ |\___
508 * | \___ | \___
509 * | \ | \
510 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
511 * -> time
512 *
513 * Returns:
514 * The watermark in bytes
515 */
516static unsigned int intel_wm_method2(unsigned int pixel_rate,
517 unsigned int htotal,
518 unsigned int width,
519 unsigned int cpp,
520 unsigned int latency)
521{
522 unsigned int ret;
523
524 /*
525 * FIXME remove once all users are computing
526 * watermarks in the correct place.
527 */
528 if (WARN_ON_ONCE(htotal == 0))
529 htotal = 1;
530
531 ret = (latency * pixel_rate) / (htotal * 10000);
532 ret = (ret + 1) * width * cpp;
533
534 return ret;
535}
536
537/**
538 * intel_calculate_wm - calculate watermark level
539 * @i915: the device
540 * @pixel_rate: pixel clock
541 * @wm: chip FIFO params
542 * @fifo_size: size of the FIFO buffer
543 * @cpp: bytes per pixel
544 * @latency_ns: memory latency for the platform
545 *
546 * Calculate the watermark level (the level at which the display plane will
547 * start fetching from memory again). Each chip has a different display
548 * FIFO size and allocation, so the caller needs to figure that out and pass
549 * in the correct intel_watermark_params structure.
550 *
551 * As the pixel clock runs, the FIFO will be drained at a rate that depends
552 * on the pixel size. When it reaches the watermark level, it'll start
553 * fetching FIFO line sized based chunks from memory until the FIFO fills
554 * past the watermark point. If the FIFO drains completely, a FIFO underrun
555 * will occur, and a display engine hang could result.
556 */
557static unsigned int intel_calculate_wm(struct drm_i915_private *i915,
558 int pixel_rate,
559 const struct intel_watermark_params *wm,
560 int fifo_size, int cpp,
561 unsigned int latency_ns)
562{
563 int entries, wm_size;
564
565 /*
566 * Note: we need to make sure we don't overflow for various clock &
567 * latency values.
568 * clocks go from a few thousand to several hundred thousand.
569 * latency is usually a few thousand
570 */
571 entries = intel_wm_method1(pixel_rate, cpp,
572 latency_ns / 100);
573 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
574 wm->guard_size;
575 drm_dbg_kms(&i915->drm, "FIFO entries required for mode: %d\n", entries);
576
577 wm_size = fifo_size - entries;
578 drm_dbg_kms(&i915->drm, "FIFO watermark level: %d\n", wm_size);
579
580 /* Don't promote wm_size to unsigned... */
581 if (wm_size > wm->max_wm)
582 wm_size = wm->max_wm;
583 if (wm_size <= 0)
584 wm_size = wm->default_wm;
585
586 /*
587 * Bspec seems to indicate that the value shouldn't be lower than
588 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
589 * Lets go for 8 which is the burst size since certain platforms
590 * already use a hardcoded 8 (which is what the spec says should be
591 * done).
592 */
593 if (wm_size <= 8)
594 wm_size = 8;
595
596 return wm_size;
597}
598
599static bool is_disabling(int old, int new, int threshold)
600{
601 return old >= threshold && new < threshold;
602}
603
604static bool is_enabling(int old, int new, int threshold)
605{
606 return old < threshold && new >= threshold;
607}
608
609static bool intel_crtc_active(struct intel_crtc *crtc)
610{
611 /* Be paranoid as we can arrive here with only partial
612 * state retrieved from the hardware during setup.
613 *
614 * We can ditch the adjusted_mode.crtc_clock check as soon
615 * as Haswell has gained clock readout/fastboot support.
616 *
617 * We can ditch the crtc->primary->state->fb check as soon as we can
618 * properly reconstruct framebuffers.
619 *
620 * FIXME: The intel_crtc->active here should be switched to
621 * crtc->state->active once we have proper CRTC states wired up
622 * for atomic.
623 */
624 return crtc->active && crtc->base.primary->state->fb &&
625 crtc->config->hw.adjusted_mode.crtc_clock;
626}
627
628static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
629{
630 struct intel_crtc *crtc, *enabled = NULL;
631
632 for_each_intel_crtc(&dev_priv->drm, crtc) {
633 if (intel_crtc_active(crtc)) {
634 if (enabled)
635 return NULL;
636 enabled = crtc;
637 }
638 }
639
640 return enabled;
641}
642
643static void pnv_update_wm(struct drm_i915_private *dev_priv)
644{
645 struct intel_crtc *crtc;
646 const struct cxsr_latency *latency;
647 u32 reg;
648 unsigned int wm;
649
650 latency = pnv_get_cxsr_latency(dev_priv);
651 if (!latency) {
652 drm_dbg_kms(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
653 intel_set_memory_cxsr(dev_priv, false);
654 return;
655 }
656
657 crtc = single_enabled_crtc(dev_priv);
658 if (crtc) {
659 const struct drm_framebuffer *fb =
660 crtc->base.primary->state->fb;
661 int pixel_rate = crtc->config->pixel_rate;
662 int cpp = fb->format->cpp[0];
663
664 /* Display SR */
665 wm = intel_calculate_wm(dev_priv, pixel_rate,
666 &pnv_display_wm,
667 pnv_display_wm.fifo_size,
668 cpp, latency->display_sr);
669 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
670 reg &= ~DSPFW_SR_MASK;
671 reg |= FW_WM(wm, SR);
672 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv), reg);
673 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
674
675 /* cursor SR */
676 wm = intel_calculate_wm(dev_priv, pixel_rate,
677 &pnv_cursor_wm,
678 pnv_display_wm.fifo_size,
679 4, latency->cursor_sr);
680 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
681 DSPFW_CURSOR_SR_MASK,
682 FW_WM(wm, CURSOR_SR));
683
684 /* Display HPLL off SR */
685 wm = intel_calculate_wm(dev_priv, pixel_rate,
686 &pnv_display_hplloff_wm,
687 pnv_display_hplloff_wm.fifo_size,
688 cpp, latency->display_hpll_disable);
689 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
690 DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
691
692 /* cursor HPLL off SR */
693 wm = intel_calculate_wm(dev_priv, pixel_rate,
694 &pnv_cursor_hplloff_wm,
695 pnv_display_hplloff_wm.fifo_size,
696 4, latency->cursor_hpll_disable);
697 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
698 reg &= ~DSPFW_HPLL_CURSOR_MASK;
699 reg |= FW_WM(wm, HPLL_CURSOR);
700 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), reg);
701 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
702
703 intel_set_memory_cxsr(dev_priv, true);
704 } else {
705 intel_set_memory_cxsr(dev_priv, false);
706 }
707}
708
709static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state,
710 const struct intel_plane_state *new_plane_state)
711{
712 /* Update watermarks on tiling or size changes. */
713 if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)
714 return true;
715
716 if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)
717 return false;
718
719 if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||
720 old_plane_state->hw.rotation != new_plane_state->hw.rotation ||
721 drm_rect_width(&old_plane_state->uapi.src) != drm_rect_width(&new_plane_state->uapi.src) ||
722 drm_rect_height(&old_plane_state->uapi.src) != drm_rect_height(&new_plane_state->uapi.src) ||
723 drm_rect_width(&old_plane_state->uapi.dst) != drm_rect_width(&new_plane_state->uapi.dst) ||
724 drm_rect_height(&old_plane_state->uapi.dst) != drm_rect_height(&new_plane_state->uapi.dst))
725 return true;
726
727 return false;
728}
729
730static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state,
731 const struct intel_plane_state *old_plane_state,
732 const struct intel_plane_state *new_plane_state)
733{
734 bool turn_off, turn_on, visible, was_visible, mode_changed;
735
736 mode_changed = intel_crtc_needs_modeset(new_crtc_state);
737 was_visible = old_plane_state->uapi.visible;
738 visible = new_plane_state->uapi.visible;
739
740 if (!was_visible && !visible)
741 return;
742
743 turn_off = was_visible && (!visible || mode_changed);
744 turn_on = visible && (!was_visible || mode_changed);
745
746 /* FIXME nuke when all wm code is atomic */
747 if (turn_on) {
748 new_crtc_state->update_wm_pre = true;
749 } else if (turn_off) {
750 new_crtc_state->update_wm_post = true;
751 } else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) {
752 /* FIXME bollocks */
753 new_crtc_state->update_wm_pre = true;
754 new_crtc_state->update_wm_post = true;
755 }
756}
757
758static int i9xx_compute_watermarks(struct intel_atomic_state *state,
759 struct intel_crtc *crtc)
760{
761 struct intel_crtc_state *new_crtc_state =
762 intel_atomic_get_new_crtc_state(state, crtc);
763 const struct intel_plane_state *old_plane_state;
764 const struct intel_plane_state *new_plane_state;
765 struct intel_plane *plane;
766 int i;
767
768 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
769 new_plane_state, i) {
770 if (plane->pipe != crtc->pipe)
771 continue;
772
773 i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state);
774 }
775
776 return 0;
777}
778
779/*
780 * Documentation says:
781 * "If the line size is small, the TLB fetches can get in the way of the
782 * data fetches, causing some lag in the pixel data return which is not
783 * accounted for in the above formulas. The following adjustment only
784 * needs to be applied if eight whole lines fit in the buffer at once.
785 * The WM is adjusted upwards by the difference between the FIFO size
786 * and the size of 8 whole lines. This adjustment is always performed
787 * in the actual pixel depth regardless of whether FBC is enabled or not."
788 */
789static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
790{
791 int tlb_miss = fifo_size * 64 - width * cpp * 8;
792
793 return max(0, tlb_miss);
794}
795
796static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
797 const struct g4x_wm_values *wm)
798{
799 struct intel_display *display = &dev_priv->display;
800 enum pipe pipe;
801
802 for_each_pipe(dev_priv, pipe)
803 trace_g4x_wm(intel_crtc_for_pipe(display, pipe), wm);
804
805 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
806 FW_WM(wm->sr.plane, SR) |
807 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
808 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
809 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
810 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
811 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
812 FW_WM(wm->sr.fbc, FBC_SR) |
813 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
814 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
815 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
816 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
817 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
818 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
819 FW_WM(wm->sr.cursor, CURSOR_SR) |
820 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
821 FW_WM(wm->hpll.plane, HPLL_SR));
822
823 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
824}
825
826#define FW_WM_VLV(value, plane) \
827 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
828
829static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
830 const struct vlv_wm_values *wm)
831{
832 struct intel_display *display = &dev_priv->display;
833 enum pipe pipe;
834
835 for_each_pipe(dev_priv, pipe) {
836 trace_vlv_wm(intel_crtc_for_pipe(display, pipe), wm);
837
838 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
839 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
840 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
841 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
842 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
843 }
844
845 /*
846 * Zero the (unused) WM1 watermarks, and also clear all the
847 * high order bits so that there are no out of bounds values
848 * present in the registers during the reprogramming.
849 */
850 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
851 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
852 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
853 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
854 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
855
856 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
857 FW_WM(wm->sr.plane, SR) |
858 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
859 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
860 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
861 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
862 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
863 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
864 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
865 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
866 FW_WM(wm->sr.cursor, CURSOR_SR));
867
868 if (IS_CHERRYVIEW(dev_priv)) {
869 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
870 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
871 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
872 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
873 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
874 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
875 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
876 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
877 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
878 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
879 FW_WM(wm->sr.plane >> 9, SR_HI) |
880 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
881 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
882 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
883 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
884 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
885 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
886 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
887 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
888 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
889 } else {
890 intel_uncore_write(&dev_priv->uncore, DSPFW7,
891 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
892 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
893 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
894 FW_WM(wm->sr.plane >> 9, SR_HI) |
895 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
896 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
897 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
898 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
899 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
900 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
901 }
902
903 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
904}
905
906#undef FW_WM_VLV
907
908static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
909{
910 /* all latencies in usec */
911 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
912 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
913 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
914
915 dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
916}
917
918static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
919{
920 /*
921 * DSPCNTR[13] supposedly controls whether the
922 * primary plane can use the FIFO space otherwise
923 * reserved for the sprite plane. It's not 100% clear
924 * what the actual FIFO size is, but it looks like we
925 * can happily set both primary and sprite watermarks
926 * up to 127 cachelines. So that would seem to mean
927 * that either DSPCNTR[13] doesn't do anything, or that
928 * the total FIFO is >= 256 cachelines in size. Either
929 * way, we don't seem to have to worry about this
930 * repartitioning as the maximum watermark value the
931 * register can hold for each plane is lower than the
932 * minimum FIFO size.
933 */
934 switch (plane_id) {
935 case PLANE_CURSOR:
936 return 63;
937 case PLANE_PRIMARY:
938 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
939 case PLANE_SPRITE0:
940 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
941 default:
942 MISSING_CASE(plane_id);
943 return 0;
944 }
945}
946
947static int g4x_fbc_fifo_size(int level)
948{
949 switch (level) {
950 case G4X_WM_LEVEL_SR:
951 return 7;
952 case G4X_WM_LEVEL_HPLL:
953 return 15;
954 default:
955 MISSING_CASE(level);
956 return 0;
957 }
958}
959
960static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
961 const struct intel_plane_state *plane_state,
962 int level)
963{
964 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
965 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
966 const struct drm_display_mode *pipe_mode =
967 &crtc_state->hw.pipe_mode;
968 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
969 unsigned int pixel_rate, htotal, cpp, width, wm;
970
971 if (latency == 0)
972 return USHRT_MAX;
973
974 if (!intel_wm_plane_visible(crtc_state, plane_state))
975 return 0;
976
977 cpp = plane_state->hw.fb->format->cpp[0];
978
979 /*
980 * WaUse32BppForSRWM:ctg,elk
981 *
982 * The spec fails to list this restriction for the
983 * HPLL watermark, which seems a little strange.
984 * Let's use 32bpp for the HPLL watermark as well.
985 */
986 if (plane->id == PLANE_PRIMARY &&
987 level != G4X_WM_LEVEL_NORMAL)
988 cpp = max(cpp, 4u);
989
990 pixel_rate = crtc_state->pixel_rate;
991 htotal = pipe_mode->crtc_htotal;
992 width = drm_rect_width(&plane_state->uapi.src) >> 16;
993
994 if (plane->id == PLANE_CURSOR) {
995 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
996 } else if (plane->id == PLANE_PRIMARY &&
997 level == G4X_WM_LEVEL_NORMAL) {
998 wm = intel_wm_method1(pixel_rate, cpp, latency);
999 } else {
1000 unsigned int small, large;
1001
1002 small = intel_wm_method1(pixel_rate, cpp, latency);
1003 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1004
1005 wm = min(small, large);
1006 }
1007
1008 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1009 width, cpp);
1010
1011 wm = DIV_ROUND_UP(wm, 64) + 2;
1012
1013 return min_t(unsigned int, wm, USHRT_MAX);
1014}
1015
1016static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1017 int level, enum plane_id plane_id, u16 value)
1018{
1019 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1020 bool dirty = false;
1021
1022 for (; level < dev_priv->display.wm.num_levels; level++) {
1023 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1024
1025 dirty |= raw->plane[plane_id] != value;
1026 raw->plane[plane_id] = value;
1027 }
1028
1029 return dirty;
1030}
1031
1032static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1033 int level, u16 value)
1034{
1035 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1036 bool dirty = false;
1037
1038 /* NORMAL level doesn't have an FBC watermark */
1039 level = max(level, G4X_WM_LEVEL_SR);
1040
1041 for (; level < dev_priv->display.wm.num_levels; level++) {
1042 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1043
1044 dirty |= raw->fbc != value;
1045 raw->fbc = value;
1046 }
1047
1048 return dirty;
1049}
1050
1051static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1052 const struct intel_plane_state *plane_state,
1053 u32 pri_val);
1054
1055static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1056 const struct intel_plane_state *plane_state)
1057{
1058 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1059 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1060 enum plane_id plane_id = plane->id;
1061 bool dirty = false;
1062 int level;
1063
1064 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1065 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1066 if (plane_id == PLANE_PRIMARY)
1067 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1068 goto out;
1069 }
1070
1071 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1072 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1073 int wm, max_wm;
1074
1075 wm = g4x_compute_wm(crtc_state, plane_state, level);
1076 max_wm = g4x_plane_fifo_size(plane_id, level);
1077
1078 if (wm > max_wm)
1079 break;
1080
1081 dirty |= raw->plane[plane_id] != wm;
1082 raw->plane[plane_id] = wm;
1083
1084 if (plane_id != PLANE_PRIMARY ||
1085 level == G4X_WM_LEVEL_NORMAL)
1086 continue;
1087
1088 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1089 raw->plane[plane_id]);
1090 max_wm = g4x_fbc_fifo_size(level);
1091
1092 /*
1093 * FBC wm is not mandatory as we
1094 * can always just disable its use.
1095 */
1096 if (wm > max_wm)
1097 wm = USHRT_MAX;
1098
1099 dirty |= raw->fbc != wm;
1100 raw->fbc = wm;
1101 }
1102
1103 /* mark watermarks as invalid */
1104 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1105
1106 if (plane_id == PLANE_PRIMARY)
1107 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1108
1109 out:
1110 if (dirty) {
1111 drm_dbg_kms(&dev_priv->drm,
1112 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1113 plane->base.name,
1114 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1115 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1116 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1117
1118 if (plane_id == PLANE_PRIMARY)
1119 drm_dbg_kms(&dev_priv->drm,
1120 "FBC watermarks: SR=%d, HPLL=%d\n",
1121 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1122 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1123 }
1124
1125 return dirty;
1126}
1127
1128static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1129 enum plane_id plane_id, int level)
1130{
1131 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1132
1133 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1134}
1135
1136static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1137 int level)
1138{
1139 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1140
1141 if (level >= dev_priv->display.wm.num_levels)
1142 return false;
1143
1144 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1145 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1146 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1147}
1148
1149/* mark all levels starting from 'level' as invalid */
1150static void g4x_invalidate_wms(struct intel_crtc *crtc,
1151 struct g4x_wm_state *wm_state, int level)
1152{
1153 if (level <= G4X_WM_LEVEL_NORMAL) {
1154 enum plane_id plane_id;
1155
1156 for_each_plane_id_on_crtc(crtc, plane_id)
1157 wm_state->wm.plane[plane_id] = USHRT_MAX;
1158 }
1159
1160 if (level <= G4X_WM_LEVEL_SR) {
1161 wm_state->cxsr = false;
1162 wm_state->sr.cursor = USHRT_MAX;
1163 wm_state->sr.plane = USHRT_MAX;
1164 wm_state->sr.fbc = USHRT_MAX;
1165 }
1166
1167 if (level <= G4X_WM_LEVEL_HPLL) {
1168 wm_state->hpll_en = false;
1169 wm_state->hpll.cursor = USHRT_MAX;
1170 wm_state->hpll.plane = USHRT_MAX;
1171 wm_state->hpll.fbc = USHRT_MAX;
1172 }
1173}
1174
1175static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1176 int level)
1177{
1178 if (level < G4X_WM_LEVEL_SR)
1179 return false;
1180
1181 if (level >= G4X_WM_LEVEL_SR &&
1182 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1183 return false;
1184
1185 if (level >= G4X_WM_LEVEL_HPLL &&
1186 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1187 return false;
1188
1189 return true;
1190}
1191
1192static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1193{
1194 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1195 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1196 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1197 const struct g4x_pipe_wm *raw;
1198 enum plane_id plane_id;
1199 int level;
1200
1201 level = G4X_WM_LEVEL_NORMAL;
1202 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1203 goto out;
1204
1205 raw = &crtc_state->wm.g4x.raw[level];
1206 for_each_plane_id_on_crtc(crtc, plane_id)
1207 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1208
1209 level = G4X_WM_LEVEL_SR;
1210 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1211 goto out;
1212
1213 raw = &crtc_state->wm.g4x.raw[level];
1214 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1215 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1216 wm_state->sr.fbc = raw->fbc;
1217
1218 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1219
1220 level = G4X_WM_LEVEL_HPLL;
1221 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1222 goto out;
1223
1224 raw = &crtc_state->wm.g4x.raw[level];
1225 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1226 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1227 wm_state->hpll.fbc = raw->fbc;
1228
1229 wm_state->hpll_en = wm_state->cxsr;
1230
1231 level++;
1232
1233 out:
1234 if (level == G4X_WM_LEVEL_NORMAL)
1235 return -EINVAL;
1236
1237 /* invalidate the higher levels */
1238 g4x_invalidate_wms(crtc, wm_state, level);
1239
1240 /*
1241 * Determine if the FBC watermark(s) can be used. IF
1242 * this isn't the case we prefer to disable the FBC
1243 * watermark(s) rather than disable the SR/HPLL
1244 * level(s) entirely. 'level-1' is the highest valid
1245 * level here.
1246 */
1247 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1248
1249 return 0;
1250}
1251
1252static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1253 struct intel_crtc *crtc)
1254{
1255 struct intel_crtc_state *crtc_state =
1256 intel_atomic_get_new_crtc_state(state, crtc);
1257 const struct intel_plane_state *old_plane_state;
1258 const struct intel_plane_state *new_plane_state;
1259 struct intel_plane *plane;
1260 unsigned int dirty = 0;
1261 int i;
1262
1263 for_each_oldnew_intel_plane_in_state(state, plane,
1264 old_plane_state,
1265 new_plane_state, i) {
1266 if (new_plane_state->hw.crtc != &crtc->base &&
1267 old_plane_state->hw.crtc != &crtc->base)
1268 continue;
1269
1270 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1271 dirty |= BIT(plane->id);
1272 }
1273
1274 if (!dirty)
1275 return 0;
1276
1277 return _g4x_compute_pipe_wm(crtc_state);
1278}
1279
1280static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1281 struct intel_crtc *crtc)
1282{
1283 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1284 struct intel_crtc_state *new_crtc_state =
1285 intel_atomic_get_new_crtc_state(state, crtc);
1286 const struct intel_crtc_state *old_crtc_state =
1287 intel_atomic_get_old_crtc_state(state, crtc);
1288 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1289 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1290 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1291 enum plane_id plane_id;
1292
1293 if (!new_crtc_state->hw.active ||
1294 intel_crtc_needs_modeset(new_crtc_state)) {
1295 *intermediate = *optimal;
1296
1297 intermediate->cxsr = false;
1298 intermediate->hpll_en = false;
1299 goto out;
1300 }
1301
1302 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1303 !new_crtc_state->disable_cxsr;
1304 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1305 !new_crtc_state->disable_cxsr;
1306 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1307
1308 for_each_plane_id_on_crtc(crtc, plane_id) {
1309 intermediate->wm.plane[plane_id] =
1310 max(optimal->wm.plane[plane_id],
1311 active->wm.plane[plane_id]);
1312
1313 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1314 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1315 }
1316
1317 intermediate->sr.plane = max(optimal->sr.plane,
1318 active->sr.plane);
1319 intermediate->sr.cursor = max(optimal->sr.cursor,
1320 active->sr.cursor);
1321 intermediate->sr.fbc = max(optimal->sr.fbc,
1322 active->sr.fbc);
1323
1324 intermediate->hpll.plane = max(optimal->hpll.plane,
1325 active->hpll.plane);
1326 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1327 active->hpll.cursor);
1328 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1329 active->hpll.fbc);
1330
1331 drm_WARN_ON(&dev_priv->drm,
1332 (intermediate->sr.plane >
1333 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1334 intermediate->sr.cursor >
1335 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1336 intermediate->cxsr);
1337 drm_WARN_ON(&dev_priv->drm,
1338 (intermediate->sr.plane >
1339 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1340 intermediate->sr.cursor >
1341 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1342 intermediate->hpll_en);
1343
1344 drm_WARN_ON(&dev_priv->drm,
1345 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1346 intermediate->fbc_en && intermediate->cxsr);
1347 drm_WARN_ON(&dev_priv->drm,
1348 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1349 intermediate->fbc_en && intermediate->hpll_en);
1350
1351out:
1352 /*
1353 * If our intermediate WM are identical to the final WM, then we can
1354 * omit the post-vblank programming; only update if it's different.
1355 */
1356 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1357 new_crtc_state->wm.need_postvbl_update = true;
1358
1359 return 0;
1360}
1361
1362static int g4x_compute_watermarks(struct intel_atomic_state *state,
1363 struct intel_crtc *crtc)
1364{
1365 int ret;
1366
1367 ret = g4x_compute_pipe_wm(state, crtc);
1368 if (ret)
1369 return ret;
1370
1371 ret = g4x_compute_intermediate_wm(state, crtc);
1372 if (ret)
1373 return ret;
1374
1375 return 0;
1376}
1377
1378static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1379 struct g4x_wm_values *wm)
1380{
1381 struct intel_crtc *crtc;
1382 int num_active_pipes = 0;
1383
1384 wm->cxsr = true;
1385 wm->hpll_en = true;
1386 wm->fbc_en = true;
1387
1388 for_each_intel_crtc(&dev_priv->drm, crtc) {
1389 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1390
1391 if (!crtc->active)
1392 continue;
1393
1394 if (!wm_state->cxsr)
1395 wm->cxsr = false;
1396 if (!wm_state->hpll_en)
1397 wm->hpll_en = false;
1398 if (!wm_state->fbc_en)
1399 wm->fbc_en = false;
1400
1401 num_active_pipes++;
1402 }
1403
1404 if (num_active_pipes != 1) {
1405 wm->cxsr = false;
1406 wm->hpll_en = false;
1407 wm->fbc_en = false;
1408 }
1409
1410 for_each_intel_crtc(&dev_priv->drm, crtc) {
1411 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1412 enum pipe pipe = crtc->pipe;
1413
1414 wm->pipe[pipe] = wm_state->wm;
1415 if (crtc->active && wm->cxsr)
1416 wm->sr = wm_state->sr;
1417 if (crtc->active && wm->hpll_en)
1418 wm->hpll = wm_state->hpll;
1419 }
1420}
1421
1422static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1423{
1424 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1425 struct g4x_wm_values new_wm = {};
1426
1427 g4x_merge_wm(dev_priv, &new_wm);
1428
1429 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1430 return;
1431
1432 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1433 _intel_set_memory_cxsr(dev_priv, false);
1434
1435 g4x_write_wm_values(dev_priv, &new_wm);
1436
1437 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1438 _intel_set_memory_cxsr(dev_priv, true);
1439
1440 *old_wm = new_wm;
1441}
1442
1443static void g4x_initial_watermarks(struct intel_atomic_state *state,
1444 struct intel_crtc *crtc)
1445{
1446 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1447 const struct intel_crtc_state *crtc_state =
1448 intel_atomic_get_new_crtc_state(state, crtc);
1449
1450 mutex_lock(&dev_priv->display.wm.wm_mutex);
1451 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1452 g4x_program_watermarks(dev_priv);
1453 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1454}
1455
1456static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1457 struct intel_crtc *crtc)
1458{
1459 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1460 const struct intel_crtc_state *crtc_state =
1461 intel_atomic_get_new_crtc_state(state, crtc);
1462
1463 if (!crtc_state->wm.need_postvbl_update)
1464 return;
1465
1466 mutex_lock(&dev_priv->display.wm.wm_mutex);
1467 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1468 g4x_program_watermarks(dev_priv);
1469 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1470}
1471
1472/* latency must be in 0.1us units. */
1473static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1474 unsigned int htotal,
1475 unsigned int width,
1476 unsigned int cpp,
1477 unsigned int latency)
1478{
1479 unsigned int ret;
1480
1481 ret = intel_wm_method2(pixel_rate, htotal,
1482 width, cpp, latency);
1483 ret = DIV_ROUND_UP(ret, 64);
1484
1485 return ret;
1486}
1487
1488static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1489{
1490 /* all latencies in usec */
1491 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1492
1493 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1494
1495 if (IS_CHERRYVIEW(dev_priv)) {
1496 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1497 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1498
1499 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1500 }
1501}
1502
1503static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1504 const struct intel_plane_state *plane_state,
1505 int level)
1506{
1507 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1508 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1509 const struct drm_display_mode *pipe_mode =
1510 &crtc_state->hw.pipe_mode;
1511 unsigned int pixel_rate, htotal, cpp, width, wm;
1512
1513 if (dev_priv->display.wm.pri_latency[level] == 0)
1514 return USHRT_MAX;
1515
1516 if (!intel_wm_plane_visible(crtc_state, plane_state))
1517 return 0;
1518
1519 cpp = plane_state->hw.fb->format->cpp[0];
1520 pixel_rate = crtc_state->pixel_rate;
1521 htotal = pipe_mode->crtc_htotal;
1522 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1523
1524 if (plane->id == PLANE_CURSOR) {
1525 /*
1526 * FIXME the formula gives values that are
1527 * too big for the cursor FIFO, and hence we
1528 * would never be able to use cursors. For
1529 * now just hardcode the watermark.
1530 */
1531 wm = 63;
1532 } else {
1533 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1534 dev_priv->display.wm.pri_latency[level] * 10);
1535 }
1536
1537 return min_t(unsigned int, wm, USHRT_MAX);
1538}
1539
1540static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1541{
1542 return (active_planes & (BIT(PLANE_SPRITE0) |
1543 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1544}
1545
1546static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1547{
1548 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1549 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1550 const struct g4x_pipe_wm *raw =
1551 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1552 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1553 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1554 int num_active_planes = hweight8(active_planes);
1555 const int fifo_size = 511;
1556 int fifo_extra, fifo_left = fifo_size;
1557 int sprite0_fifo_extra = 0;
1558 unsigned int total_rate;
1559 enum plane_id plane_id;
1560
1561 /*
1562 * When enabling sprite0 after sprite1 has already been enabled
1563 * we tend to get an underrun unless sprite0 already has some
1564 * FIFO space allcoated. Hence we always allocate at least one
1565 * cacheline for sprite0 whenever sprite1 is enabled.
1566 *
1567 * All other plane enable sequences appear immune to this problem.
1568 */
1569 if (vlv_need_sprite0_fifo_workaround(active_planes))
1570 sprite0_fifo_extra = 1;
1571
1572 total_rate = raw->plane[PLANE_PRIMARY] +
1573 raw->plane[PLANE_SPRITE0] +
1574 raw->plane[PLANE_SPRITE1] +
1575 sprite0_fifo_extra;
1576
1577 if (total_rate > fifo_size)
1578 return -EINVAL;
1579
1580 if (total_rate == 0)
1581 total_rate = 1;
1582
1583 for_each_plane_id_on_crtc(crtc, plane_id) {
1584 unsigned int rate;
1585
1586 if ((active_planes & BIT(plane_id)) == 0) {
1587 fifo_state->plane[plane_id] = 0;
1588 continue;
1589 }
1590
1591 rate = raw->plane[plane_id];
1592 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1593 fifo_left -= fifo_state->plane[plane_id];
1594 }
1595
1596 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1597 fifo_left -= sprite0_fifo_extra;
1598
1599 fifo_state->plane[PLANE_CURSOR] = 63;
1600
1601 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1602
1603 /* spread the remainder evenly */
1604 for_each_plane_id_on_crtc(crtc, plane_id) {
1605 int plane_extra;
1606
1607 if (fifo_left == 0)
1608 break;
1609
1610 if ((active_planes & BIT(plane_id)) == 0)
1611 continue;
1612
1613 plane_extra = min(fifo_extra, fifo_left);
1614 fifo_state->plane[plane_id] += plane_extra;
1615 fifo_left -= plane_extra;
1616 }
1617
1618 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1619
1620 /* give it all to the first plane if none are active */
1621 if (active_planes == 0) {
1622 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1623 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1624 }
1625
1626 return 0;
1627}
1628
1629/* mark all levels starting from 'level' as invalid */
1630static void vlv_invalidate_wms(struct intel_crtc *crtc,
1631 struct vlv_wm_state *wm_state, int level)
1632{
1633 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1634
1635 for (; level < dev_priv->display.wm.num_levels; level++) {
1636 enum plane_id plane_id;
1637
1638 for_each_plane_id_on_crtc(crtc, plane_id)
1639 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1640
1641 wm_state->sr[level].cursor = USHRT_MAX;
1642 wm_state->sr[level].plane = USHRT_MAX;
1643 }
1644}
1645
1646static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1647{
1648 if (wm > fifo_size)
1649 return USHRT_MAX;
1650 else
1651 return fifo_size - wm;
1652}
1653
1654/*
1655 * Starting from 'level' set all higher
1656 * levels to 'value' in the "raw" watermarks.
1657 */
1658static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1659 int level, enum plane_id plane_id, u16 value)
1660{
1661 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1662 bool dirty = false;
1663
1664 for (; level < dev_priv->display.wm.num_levels; level++) {
1665 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1666
1667 dirty |= raw->plane[plane_id] != value;
1668 raw->plane[plane_id] = value;
1669 }
1670
1671 return dirty;
1672}
1673
1674static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1675 const struct intel_plane_state *plane_state)
1676{
1677 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1678 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1679 enum plane_id plane_id = plane->id;
1680 int level;
1681 bool dirty = false;
1682
1683 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1684 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1685 goto out;
1686 }
1687
1688 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1689 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1690 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1691 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1692
1693 if (wm > max_wm)
1694 break;
1695
1696 dirty |= raw->plane[plane_id] != wm;
1697 raw->plane[plane_id] = wm;
1698 }
1699
1700 /* mark all higher levels as invalid */
1701 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1702
1703out:
1704 if (dirty)
1705 drm_dbg_kms(&dev_priv->drm,
1706 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1707 plane->base.name,
1708 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1709 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1710 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1711
1712 return dirty;
1713}
1714
1715static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1716 enum plane_id plane_id, int level)
1717{
1718 const struct g4x_pipe_wm *raw =
1719 &crtc_state->wm.vlv.raw[level];
1720 const struct vlv_fifo_state *fifo_state =
1721 &crtc_state->wm.vlv.fifo_state;
1722
1723 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1724}
1725
1726static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1727{
1728 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1729 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1730 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1731 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1732}
1733
1734static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1735{
1736 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1737 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1738 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1739 const struct vlv_fifo_state *fifo_state =
1740 &crtc_state->wm.vlv.fifo_state;
1741 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1742 int num_active_planes = hweight8(active_planes);
1743 enum plane_id plane_id;
1744 int level;
1745
1746 /* initially allow all levels */
1747 wm_state->num_levels = dev_priv->display.wm.num_levels;
1748 /*
1749 * Note that enabling cxsr with no primary/sprite planes
1750 * enabled can wedge the pipe. Hence we only allow cxsr
1751 * with exactly one enabled primary/sprite plane.
1752 */
1753 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1754
1755 for (level = 0; level < wm_state->num_levels; level++) {
1756 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1757 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1758
1759 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1760 break;
1761
1762 for_each_plane_id_on_crtc(crtc, plane_id) {
1763 wm_state->wm[level].plane[plane_id] =
1764 vlv_invert_wm_value(raw->plane[plane_id],
1765 fifo_state->plane[plane_id]);
1766 }
1767
1768 wm_state->sr[level].plane =
1769 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1770 raw->plane[PLANE_SPRITE0],
1771 raw->plane[PLANE_SPRITE1]),
1772 sr_fifo_size);
1773
1774 wm_state->sr[level].cursor =
1775 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1776 63);
1777 }
1778
1779 if (level == 0)
1780 return -EINVAL;
1781
1782 /* limit to only levels we can actually handle */
1783 wm_state->num_levels = level;
1784
1785 /* invalidate the higher levels */
1786 vlv_invalidate_wms(crtc, wm_state, level);
1787
1788 return 0;
1789}
1790
1791static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1792 struct intel_crtc *crtc)
1793{
1794 struct intel_crtc_state *crtc_state =
1795 intel_atomic_get_new_crtc_state(state, crtc);
1796 const struct intel_plane_state *old_plane_state;
1797 const struct intel_plane_state *new_plane_state;
1798 struct intel_plane *plane;
1799 unsigned int dirty = 0;
1800 int i;
1801
1802 for_each_oldnew_intel_plane_in_state(state, plane,
1803 old_plane_state,
1804 new_plane_state, i) {
1805 if (new_plane_state->hw.crtc != &crtc->base &&
1806 old_plane_state->hw.crtc != &crtc->base)
1807 continue;
1808
1809 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1810 dirty |= BIT(plane->id);
1811 }
1812
1813 /*
1814 * DSPARB registers may have been reset due to the
1815 * power well being turned off. Make sure we restore
1816 * them to a consistent state even if no primary/sprite
1817 * planes are initially active. We also force a FIFO
1818 * recomputation so that we are sure to sanitize the
1819 * FIFO setting we took over from the BIOS even if there
1820 * are no active planes on the crtc.
1821 */
1822 if (intel_crtc_needs_modeset(crtc_state))
1823 dirty = ~0;
1824
1825 if (!dirty)
1826 return 0;
1827
1828 /* cursor changes don't warrant a FIFO recompute */
1829 if (dirty & ~BIT(PLANE_CURSOR)) {
1830 const struct intel_crtc_state *old_crtc_state =
1831 intel_atomic_get_old_crtc_state(state, crtc);
1832 const struct vlv_fifo_state *old_fifo_state =
1833 &old_crtc_state->wm.vlv.fifo_state;
1834 const struct vlv_fifo_state *new_fifo_state =
1835 &crtc_state->wm.vlv.fifo_state;
1836 int ret;
1837
1838 ret = vlv_compute_fifo(crtc_state);
1839 if (ret)
1840 return ret;
1841
1842 if (intel_crtc_needs_modeset(crtc_state) ||
1843 memcmp(old_fifo_state, new_fifo_state,
1844 sizeof(*new_fifo_state)) != 0)
1845 crtc_state->fifo_changed = true;
1846 }
1847
1848 return _vlv_compute_pipe_wm(crtc_state);
1849}
1850
1851#define VLV_FIFO(plane, value) \
1852 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1853
1854static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1855 struct intel_crtc *crtc)
1856{
1857 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1858 struct intel_uncore *uncore = &dev_priv->uncore;
1859 const struct intel_crtc_state *crtc_state =
1860 intel_atomic_get_new_crtc_state(state, crtc);
1861 const struct vlv_fifo_state *fifo_state =
1862 &crtc_state->wm.vlv.fifo_state;
1863 int sprite0_start, sprite1_start, fifo_size;
1864 u32 dsparb, dsparb2, dsparb3;
1865
1866 if (!crtc_state->fifo_changed)
1867 return;
1868
1869 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1870 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1871 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1872
1873 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1874 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1875
1876 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1877
1878 /*
1879 * uncore.lock serves a double purpose here. It allows us to
1880 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1881 * it protects the DSPARB registers from getting clobbered by
1882 * parallel updates from multiple pipes.
1883 *
1884 * intel_pipe_update_start() has already disabled interrupts
1885 * for us, so a plain spin_lock() is sufficient here.
1886 */
1887 spin_lock(&uncore->lock);
1888
1889 switch (crtc->pipe) {
1890 case PIPE_A:
1891 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1892 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1893
1894 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1895 VLV_FIFO(SPRITEB, 0xff));
1896 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1897 VLV_FIFO(SPRITEB, sprite1_start));
1898
1899 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1900 VLV_FIFO(SPRITEB_HI, 0x1));
1901 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1902 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1903
1904 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1905 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1906 break;
1907 case PIPE_B:
1908 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1909 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1910
1911 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1912 VLV_FIFO(SPRITED, 0xff));
1913 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1914 VLV_FIFO(SPRITED, sprite1_start));
1915
1916 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1917 VLV_FIFO(SPRITED_HI, 0xff));
1918 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1919 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1920
1921 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1922 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1923 break;
1924 case PIPE_C:
1925 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1926 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1927
1928 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1929 VLV_FIFO(SPRITEF, 0xff));
1930 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1931 VLV_FIFO(SPRITEF, sprite1_start));
1932
1933 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1934 VLV_FIFO(SPRITEF_HI, 0xff));
1935 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1936 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1937
1938 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1939 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1940 break;
1941 default:
1942 break;
1943 }
1944
1945 intel_uncore_posting_read_fw(uncore, DSPARB(dev_priv));
1946
1947 spin_unlock(&uncore->lock);
1948}
1949
1950#undef VLV_FIFO
1951
1952static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1953 struct intel_crtc *crtc)
1954{
1955 struct intel_crtc_state *new_crtc_state =
1956 intel_atomic_get_new_crtc_state(state, crtc);
1957 const struct intel_crtc_state *old_crtc_state =
1958 intel_atomic_get_old_crtc_state(state, crtc);
1959 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1960 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1961 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1962 int level;
1963
1964 if (!new_crtc_state->hw.active ||
1965 intel_crtc_needs_modeset(new_crtc_state)) {
1966 *intermediate = *optimal;
1967
1968 intermediate->cxsr = false;
1969 goto out;
1970 }
1971
1972 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1973 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1974 !new_crtc_state->disable_cxsr;
1975
1976 for (level = 0; level < intermediate->num_levels; level++) {
1977 enum plane_id plane_id;
1978
1979 for_each_plane_id_on_crtc(crtc, plane_id) {
1980 intermediate->wm[level].plane[plane_id] =
1981 min(optimal->wm[level].plane[plane_id],
1982 active->wm[level].plane[plane_id]);
1983 }
1984
1985 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1986 active->sr[level].plane);
1987 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1988 active->sr[level].cursor);
1989 }
1990
1991 vlv_invalidate_wms(crtc, intermediate, level);
1992
1993out:
1994 /*
1995 * If our intermediate WM are identical to the final WM, then we can
1996 * omit the post-vblank programming; only update if it's different.
1997 */
1998 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1999 new_crtc_state->wm.need_postvbl_update = true;
2000
2001 return 0;
2002}
2003
2004static int vlv_compute_watermarks(struct intel_atomic_state *state,
2005 struct intel_crtc *crtc)
2006{
2007 int ret;
2008
2009 ret = vlv_compute_pipe_wm(state, crtc);
2010 if (ret)
2011 return ret;
2012
2013 ret = vlv_compute_intermediate_wm(state, crtc);
2014 if (ret)
2015 return ret;
2016
2017 return 0;
2018}
2019
2020static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2021 struct vlv_wm_values *wm)
2022{
2023 struct intel_crtc *crtc;
2024 int num_active_pipes = 0;
2025
2026 wm->level = dev_priv->display.wm.num_levels - 1;
2027 wm->cxsr = true;
2028
2029 for_each_intel_crtc(&dev_priv->drm, crtc) {
2030 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2031
2032 if (!crtc->active)
2033 continue;
2034
2035 if (!wm_state->cxsr)
2036 wm->cxsr = false;
2037
2038 num_active_pipes++;
2039 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2040 }
2041
2042 if (num_active_pipes != 1)
2043 wm->cxsr = false;
2044
2045 if (num_active_pipes > 1)
2046 wm->level = VLV_WM_LEVEL_PM2;
2047
2048 for_each_intel_crtc(&dev_priv->drm, crtc) {
2049 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2050 enum pipe pipe = crtc->pipe;
2051
2052 wm->pipe[pipe] = wm_state->wm[wm->level];
2053 if (crtc->active && wm->cxsr)
2054 wm->sr = wm_state->sr[wm->level];
2055
2056 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2057 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2058 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2059 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2060 }
2061}
2062
2063static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2064{
2065 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2066 struct vlv_wm_values new_wm = {};
2067
2068 vlv_merge_wm(dev_priv, &new_wm);
2069
2070 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2071 return;
2072
2073 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2074 chv_set_memory_dvfs(dev_priv, false);
2075
2076 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2077 chv_set_memory_pm5(dev_priv, false);
2078
2079 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2080 _intel_set_memory_cxsr(dev_priv, false);
2081
2082 vlv_write_wm_values(dev_priv, &new_wm);
2083
2084 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2085 _intel_set_memory_cxsr(dev_priv, true);
2086
2087 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2088 chv_set_memory_pm5(dev_priv, true);
2089
2090 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2091 chv_set_memory_dvfs(dev_priv, true);
2092
2093 *old_wm = new_wm;
2094}
2095
2096static void vlv_initial_watermarks(struct intel_atomic_state *state,
2097 struct intel_crtc *crtc)
2098{
2099 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2100 const struct intel_crtc_state *crtc_state =
2101 intel_atomic_get_new_crtc_state(state, crtc);
2102
2103 mutex_lock(&dev_priv->display.wm.wm_mutex);
2104 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2105 vlv_program_watermarks(dev_priv);
2106 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2107}
2108
2109static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2110 struct intel_crtc *crtc)
2111{
2112 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2113 const struct intel_crtc_state *crtc_state =
2114 intel_atomic_get_new_crtc_state(state, crtc);
2115
2116 if (!crtc_state->wm.need_postvbl_update)
2117 return;
2118
2119 mutex_lock(&dev_priv->display.wm.wm_mutex);
2120 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2121 vlv_program_watermarks(dev_priv);
2122 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2123}
2124
2125static void i965_update_wm(struct drm_i915_private *dev_priv)
2126{
2127 struct intel_crtc *crtc;
2128 int srwm = 1;
2129 int cursor_sr = 16;
2130 bool cxsr_enabled;
2131
2132 /* Calc sr entries for one plane configs */
2133 crtc = single_enabled_crtc(dev_priv);
2134 if (crtc) {
2135 /* self-refresh has much higher latency */
2136 static const int sr_latency_ns = 12000;
2137 const struct drm_display_mode *pipe_mode =
2138 &crtc->config->hw.pipe_mode;
2139 const struct drm_framebuffer *fb =
2140 crtc->base.primary->state->fb;
2141 int pixel_rate = crtc->config->pixel_rate;
2142 int htotal = pipe_mode->crtc_htotal;
2143 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2144 int cpp = fb->format->cpp[0];
2145 int entries;
2146
2147 entries = intel_wm_method2(pixel_rate, htotal,
2148 width, cpp, sr_latency_ns / 100);
2149 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2150 srwm = I965_FIFO_SIZE - entries;
2151 if (srwm < 0)
2152 srwm = 1;
2153 srwm &= 0x1ff;
2154 drm_dbg_kms(&dev_priv->drm,
2155 "self-refresh entries: %d, wm: %d\n",
2156 entries, srwm);
2157
2158 entries = intel_wm_method2(pixel_rate, htotal,
2159 crtc->base.cursor->state->crtc_w, 4,
2160 sr_latency_ns / 100);
2161 entries = DIV_ROUND_UP(entries,
2162 i965_cursor_wm_info.cacheline_size) +
2163 i965_cursor_wm_info.guard_size;
2164
2165 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2166 if (cursor_sr > i965_cursor_wm_info.max_wm)
2167 cursor_sr = i965_cursor_wm_info.max_wm;
2168
2169 drm_dbg_kms(&dev_priv->drm,
2170 "self-refresh watermark: display plane %d "
2171 "cursor %d\n", srwm, cursor_sr);
2172
2173 cxsr_enabled = true;
2174 } else {
2175 cxsr_enabled = false;
2176 /* Turn off self refresh if both pipes are enabled */
2177 intel_set_memory_cxsr(dev_priv, false);
2178 }
2179
2180 drm_dbg_kms(&dev_priv->drm,
2181 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2182 srwm);
2183
2184 /* 965 has limitations... */
2185 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
2186 FW_WM(srwm, SR) |
2187 FW_WM(8, CURSORB) |
2188 FW_WM(8, PLANEB) |
2189 FW_WM(8, PLANEA));
2190 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
2191 FW_WM(8, CURSORA) |
2192 FW_WM(8, PLANEC_OLD));
2193 /* update cursor SR watermark */
2194 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
2195 FW_WM(cursor_sr, CURSOR_SR));
2196
2197 if (cxsr_enabled)
2198 intel_set_memory_cxsr(dev_priv, true);
2199}
2200
2201#undef FW_WM
2202
2203static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2204 enum i9xx_plane_id i9xx_plane)
2205{
2206 struct intel_display *display = &i915->display;
2207 struct intel_plane *plane;
2208
2209 for_each_intel_plane(&i915->drm, plane) {
2210 if (plane->id == PLANE_PRIMARY &&
2211 plane->i9xx_plane == i9xx_plane)
2212 return intel_crtc_for_pipe(display, plane->pipe);
2213 }
2214
2215 return NULL;
2216}
2217
2218static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2219{
2220 const struct intel_watermark_params *wm_info;
2221 u32 fwater_lo;
2222 u32 fwater_hi;
2223 int cwm, srwm = 1;
2224 int fifo_size;
2225 int planea_wm, planeb_wm;
2226 struct intel_crtc *crtc;
2227
2228 if (IS_I945GM(dev_priv))
2229 wm_info = &i945_wm_info;
2230 else if (DISPLAY_VER(dev_priv) != 2)
2231 wm_info = &i915_wm_info;
2232 else
2233 wm_info = &i830_a_wm_info;
2234
2235 if (DISPLAY_VER(dev_priv) == 2)
2236 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2237 else
2238 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2239 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2240 if (intel_crtc_active(crtc)) {
2241 const struct drm_framebuffer *fb =
2242 crtc->base.primary->state->fb;
2243 int cpp;
2244
2245 if (DISPLAY_VER(dev_priv) == 2)
2246 cpp = 4;
2247 else
2248 cpp = fb->format->cpp[0];
2249
2250 planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2251 wm_info, fifo_size, cpp,
2252 pessimal_latency_ns);
2253 } else {
2254 planea_wm = fifo_size - wm_info->guard_size;
2255 if (planea_wm > (long)wm_info->max_wm)
2256 planea_wm = wm_info->max_wm;
2257 }
2258
2259 if (DISPLAY_VER(dev_priv) == 2)
2260 wm_info = &i830_bc_wm_info;
2261
2262 if (DISPLAY_VER(dev_priv) == 2)
2263 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2264 else
2265 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2266 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2267 if (intel_crtc_active(crtc)) {
2268 const struct drm_framebuffer *fb =
2269 crtc->base.primary->state->fb;
2270 int cpp;
2271
2272 if (DISPLAY_VER(dev_priv) == 2)
2273 cpp = 4;
2274 else
2275 cpp = fb->format->cpp[0];
2276
2277 planeb_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2278 wm_info, fifo_size, cpp,
2279 pessimal_latency_ns);
2280 } else {
2281 planeb_wm = fifo_size - wm_info->guard_size;
2282 if (planeb_wm > (long)wm_info->max_wm)
2283 planeb_wm = wm_info->max_wm;
2284 }
2285
2286 drm_dbg_kms(&dev_priv->drm,
2287 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2288
2289 crtc = single_enabled_crtc(dev_priv);
2290 if (IS_I915GM(dev_priv) && crtc) {
2291 struct drm_gem_object *obj;
2292
2293 obj = intel_fb_bo(crtc->base.primary->state->fb);
2294
2295 /* self-refresh seems busted with untiled */
2296 if (!intel_bo_is_tiled(obj))
2297 crtc = NULL;
2298 }
2299
2300 /*
2301 * Overlay gets an aggressive default since video jitter is bad.
2302 */
2303 cwm = 2;
2304
2305 /* Play safe and disable self-refresh before adjusting watermarks. */
2306 intel_set_memory_cxsr(dev_priv, false);
2307
2308 /* Calc sr entries for one plane configs */
2309 if (HAS_FW_BLC(dev_priv) && crtc) {
2310 /* self-refresh has much higher latency */
2311 static const int sr_latency_ns = 6000;
2312 const struct drm_display_mode *pipe_mode =
2313 &crtc->config->hw.pipe_mode;
2314 const struct drm_framebuffer *fb =
2315 crtc->base.primary->state->fb;
2316 int pixel_rate = crtc->config->pixel_rate;
2317 int htotal = pipe_mode->crtc_htotal;
2318 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2319 int cpp;
2320 int entries;
2321
2322 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2323 cpp = 4;
2324 else
2325 cpp = fb->format->cpp[0];
2326
2327 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2328 sr_latency_ns / 100);
2329 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2330 drm_dbg_kms(&dev_priv->drm,
2331 "self-refresh entries: %d\n", entries);
2332 srwm = wm_info->fifo_size - entries;
2333 if (srwm < 0)
2334 srwm = 1;
2335
2336 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2337 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2338 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2339 else
2340 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2341 }
2342
2343 drm_dbg_kms(&dev_priv->drm,
2344 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2345 planea_wm, planeb_wm, cwm, srwm);
2346
2347 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2348 fwater_hi = (cwm & 0x1f);
2349
2350 /* Set request length to 8 cachelines per fetch */
2351 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2352 fwater_hi = fwater_hi | (1 << 8);
2353
2354 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2355 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2356
2357 if (crtc)
2358 intel_set_memory_cxsr(dev_priv, true);
2359}
2360
2361static void i845_update_wm(struct drm_i915_private *dev_priv)
2362{
2363 struct intel_crtc *crtc;
2364 u32 fwater_lo;
2365 int planea_wm;
2366
2367 crtc = single_enabled_crtc(dev_priv);
2368 if (crtc == NULL)
2369 return;
2370
2371 planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2372 &i845_wm_info,
2373 i845_get_fifo_size(dev_priv, PLANE_A),
2374 4, pessimal_latency_ns);
2375 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2376 fwater_lo |= (3<<8) | planea_wm;
2377
2378 drm_dbg_kms(&dev_priv->drm,
2379 "Setting FIFO watermarks - A: %d\n", planea_wm);
2380
2381 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2382}
2383
2384/* latency must be in 0.1us units. */
2385static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2386 unsigned int cpp,
2387 unsigned int latency)
2388{
2389 unsigned int ret;
2390
2391 ret = intel_wm_method1(pixel_rate, cpp, latency);
2392 ret = DIV_ROUND_UP(ret, 64) + 2;
2393
2394 return ret;
2395}
2396
2397/* latency must be in 0.1us units. */
2398static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2399 unsigned int htotal,
2400 unsigned int width,
2401 unsigned int cpp,
2402 unsigned int latency)
2403{
2404 unsigned int ret;
2405
2406 ret = intel_wm_method2(pixel_rate, htotal,
2407 width, cpp, latency);
2408 ret = DIV_ROUND_UP(ret, 64) + 2;
2409
2410 return ret;
2411}
2412
2413static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2414{
2415 /*
2416 * Neither of these should be possible since this function shouldn't be
2417 * called if the CRTC is off or the plane is invisible. But let's be
2418 * extra paranoid to avoid a potential divide-by-zero if we screw up
2419 * elsewhere in the driver.
2420 */
2421 if (WARN_ON(!cpp))
2422 return 0;
2423 if (WARN_ON(!horiz_pixels))
2424 return 0;
2425
2426 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2427}
2428
2429struct ilk_wm_maximums {
2430 u16 pri;
2431 u16 spr;
2432 u16 cur;
2433 u16 fbc;
2434};
2435
2436/*
2437 * For both WM_PIPE and WM_LP.
2438 * mem_value must be in 0.1us units.
2439 */
2440static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2441 const struct intel_plane_state *plane_state,
2442 u32 mem_value, bool is_lp)
2443{
2444 u32 method1, method2;
2445 int cpp;
2446
2447 if (mem_value == 0)
2448 return U32_MAX;
2449
2450 if (!intel_wm_plane_visible(crtc_state, plane_state))
2451 return 0;
2452
2453 cpp = plane_state->hw.fb->format->cpp[0];
2454
2455 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2456
2457 if (!is_lp)
2458 return method1;
2459
2460 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2461 crtc_state->hw.pipe_mode.crtc_htotal,
2462 drm_rect_width(&plane_state->uapi.src) >> 16,
2463 cpp, mem_value);
2464
2465 return min(method1, method2);
2466}
2467
2468/*
2469 * For both WM_PIPE and WM_LP.
2470 * mem_value must be in 0.1us units.
2471 */
2472static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2473 const struct intel_plane_state *plane_state,
2474 u32 mem_value)
2475{
2476 u32 method1, method2;
2477 int cpp;
2478
2479 if (mem_value == 0)
2480 return U32_MAX;
2481
2482 if (!intel_wm_plane_visible(crtc_state, plane_state))
2483 return 0;
2484
2485 cpp = plane_state->hw.fb->format->cpp[0];
2486
2487 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2488 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2489 crtc_state->hw.pipe_mode.crtc_htotal,
2490 drm_rect_width(&plane_state->uapi.src) >> 16,
2491 cpp, mem_value);
2492 return min(method1, method2);
2493}
2494
2495/*
2496 * For both WM_PIPE and WM_LP.
2497 * mem_value must be in 0.1us units.
2498 */
2499static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2500 const struct intel_plane_state *plane_state,
2501 u32 mem_value)
2502{
2503 int cpp;
2504
2505 if (mem_value == 0)
2506 return U32_MAX;
2507
2508 if (!intel_wm_plane_visible(crtc_state, plane_state))
2509 return 0;
2510
2511 cpp = plane_state->hw.fb->format->cpp[0];
2512
2513 return ilk_wm_method2(crtc_state->pixel_rate,
2514 crtc_state->hw.pipe_mode.crtc_htotal,
2515 drm_rect_width(&plane_state->uapi.src) >> 16,
2516 cpp, mem_value);
2517}
2518
2519/* Only for WM_LP. */
2520static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2521 const struct intel_plane_state *plane_state,
2522 u32 pri_val)
2523{
2524 int cpp;
2525
2526 if (!intel_wm_plane_visible(crtc_state, plane_state))
2527 return 0;
2528
2529 cpp = plane_state->hw.fb->format->cpp[0];
2530
2531 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2532 cpp);
2533}
2534
2535static unsigned int
2536ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2537{
2538 if (DISPLAY_VER(dev_priv) >= 8)
2539 return 3072;
2540 else if (DISPLAY_VER(dev_priv) >= 7)
2541 return 768;
2542 else
2543 return 512;
2544}
2545
2546static unsigned int
2547ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2548 int level, bool is_sprite)
2549{
2550 if (DISPLAY_VER(dev_priv) >= 8)
2551 /* BDW primary/sprite plane watermarks */
2552 return level == 0 ? 255 : 2047;
2553 else if (DISPLAY_VER(dev_priv) >= 7)
2554 /* IVB/HSW primary/sprite plane watermarks */
2555 return level == 0 ? 127 : 1023;
2556 else if (!is_sprite)
2557 /* ILK/SNB primary plane watermarks */
2558 return level == 0 ? 127 : 511;
2559 else
2560 /* ILK/SNB sprite plane watermarks */
2561 return level == 0 ? 63 : 255;
2562}
2563
2564static unsigned int
2565ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2566{
2567 if (DISPLAY_VER(dev_priv) >= 7)
2568 return level == 0 ? 63 : 255;
2569 else
2570 return level == 0 ? 31 : 63;
2571}
2572
2573static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2574{
2575 if (DISPLAY_VER(dev_priv) >= 8)
2576 return 31;
2577 else
2578 return 15;
2579}
2580
2581/* Calculate the maximum primary/sprite plane watermark */
2582static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2583 int level,
2584 const struct intel_wm_config *config,
2585 enum intel_ddb_partitioning ddb_partitioning,
2586 bool is_sprite)
2587{
2588 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2589
2590 /* if sprites aren't enabled, sprites get nothing */
2591 if (is_sprite && !config->sprites_enabled)
2592 return 0;
2593
2594 /* HSW allows LP1+ watermarks even with multiple pipes */
2595 if (level == 0 || config->num_pipes_active > 1) {
2596 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2597
2598 /*
2599 * For some reason the non self refresh
2600 * FIFO size is only half of the self
2601 * refresh FIFO size on ILK/SNB.
2602 */
2603 if (DISPLAY_VER(dev_priv) < 7)
2604 fifo_size /= 2;
2605 }
2606
2607 if (config->sprites_enabled) {
2608 /* level 0 is always calculated with 1:1 split */
2609 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2610 if (is_sprite)
2611 fifo_size *= 5;
2612 fifo_size /= 6;
2613 } else {
2614 fifo_size /= 2;
2615 }
2616 }
2617
2618 /* clamp to max that the registers can hold */
2619 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2620}
2621
2622/* Calculate the maximum cursor plane watermark */
2623static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2624 int level,
2625 const struct intel_wm_config *config)
2626{
2627 /* HSW LP1+ watermarks w/ multiple pipes */
2628 if (level > 0 && config->num_pipes_active > 1)
2629 return 64;
2630
2631 /* otherwise just report max that registers can hold */
2632 return ilk_cursor_wm_reg_max(dev_priv, level);
2633}
2634
2635static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2636 int level,
2637 const struct intel_wm_config *config,
2638 enum intel_ddb_partitioning ddb_partitioning,
2639 struct ilk_wm_maximums *max)
2640{
2641 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2642 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2643 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2644 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2645}
2646
2647static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2648 int level,
2649 struct ilk_wm_maximums *max)
2650{
2651 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2652 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2653 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2654 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2655}
2656
2657static bool ilk_validate_wm_level(struct drm_i915_private *i915,
2658 int level,
2659 const struct ilk_wm_maximums *max,
2660 struct intel_wm_level *result)
2661{
2662 bool ret;
2663
2664 /* already determined to be invalid? */
2665 if (!result->enable)
2666 return false;
2667
2668 result->enable = result->pri_val <= max->pri &&
2669 result->spr_val <= max->spr &&
2670 result->cur_val <= max->cur;
2671
2672 ret = result->enable;
2673
2674 /*
2675 * HACK until we can pre-compute everything,
2676 * and thus fail gracefully if LP0 watermarks
2677 * are exceeded...
2678 */
2679 if (level == 0 && !result->enable) {
2680 if (result->pri_val > max->pri)
2681 drm_dbg_kms(&i915->drm,
2682 "Primary WM%d too large %u (max %u)\n",
2683 level, result->pri_val, max->pri);
2684 if (result->spr_val > max->spr)
2685 drm_dbg_kms(&i915->drm,
2686 "Sprite WM%d too large %u (max %u)\n",
2687 level, result->spr_val, max->spr);
2688 if (result->cur_val > max->cur)
2689 drm_dbg_kms(&i915->drm,
2690 "Cursor WM%d too large %u (max %u)\n",
2691 level, result->cur_val, max->cur);
2692
2693 result->pri_val = min_t(u32, result->pri_val, max->pri);
2694 result->spr_val = min_t(u32, result->spr_val, max->spr);
2695 result->cur_val = min_t(u32, result->cur_val, max->cur);
2696 result->enable = true;
2697 }
2698
2699 return ret;
2700}
2701
2702static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2703 const struct intel_crtc *crtc,
2704 int level,
2705 struct intel_crtc_state *crtc_state,
2706 const struct intel_plane_state *pristate,
2707 const struct intel_plane_state *sprstate,
2708 const struct intel_plane_state *curstate,
2709 struct intel_wm_level *result)
2710{
2711 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2712 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2713 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2714
2715 /* WM1+ latency values stored in 0.5us units */
2716 if (level > 0) {
2717 pri_latency *= 5;
2718 spr_latency *= 5;
2719 cur_latency *= 5;
2720 }
2721
2722 if (pristate) {
2723 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2724 pri_latency, level);
2725 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2726 }
2727
2728 if (sprstate)
2729 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2730
2731 if (curstate)
2732 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2733
2734 result->enable = true;
2735}
2736
2737static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2738{
2739 u64 sskpd;
2740
2741 i915->display.wm.num_levels = 5;
2742
2743 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2744
2745 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2746 if (wm[0] == 0)
2747 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2748 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2749 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2750 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2751 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2752}
2753
2754static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2755{
2756 u32 sskpd;
2757
2758 i915->display.wm.num_levels = 4;
2759
2760 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2761
2762 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2763 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2764 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2765 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2766}
2767
2768static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2769{
2770 u32 mltr;
2771
2772 i915->display.wm.num_levels = 3;
2773
2774 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2775
2776 /* ILK primary LP0 latency is 700 ns */
2777 wm[0] = 7;
2778 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2779 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2780}
2781
2782static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2783 u16 wm[5])
2784{
2785 /* ILK sprite LP0 latency is 1300 ns */
2786 if (DISPLAY_VER(dev_priv) == 5)
2787 wm[0] = 13;
2788}
2789
2790static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2791 u16 wm[5])
2792{
2793 /* ILK cursor LP0 latency is 1300 ns */
2794 if (DISPLAY_VER(dev_priv) == 5)
2795 wm[0] = 13;
2796}
2797
2798static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2799 u16 wm[5], u16 min)
2800{
2801 int level;
2802
2803 if (wm[0] >= min)
2804 return false;
2805
2806 wm[0] = max(wm[0], min);
2807 for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2808 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2809
2810 return true;
2811}
2812
2813static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2814{
2815 bool changed;
2816
2817 /*
2818 * The BIOS provided WM memory latency values are often
2819 * inadequate for high resolution displays. Adjust them.
2820 */
2821 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2822 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2823 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2824
2825 if (!changed)
2826 return;
2827
2828 drm_dbg_kms(&dev_priv->drm,
2829 "WM latency values increased to avoid potential underruns\n");
2830 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2831 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2832 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2833}
2834
2835static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2836{
2837 /*
2838 * On some SNB machines (Thinkpad X220 Tablet at least)
2839 * LP3 usage can cause vblank interrupts to be lost.
2840 * The DEIIR bit will go high but it looks like the CPU
2841 * never gets interrupted.
2842 *
2843 * It's not clear whether other interrupt source could
2844 * be affected or if this is somehow limited to vblank
2845 * interrupts only. To play it safe we disable LP3
2846 * watermarks entirely.
2847 */
2848 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2849 dev_priv->display.wm.spr_latency[3] == 0 &&
2850 dev_priv->display.wm.cur_latency[3] == 0)
2851 return;
2852
2853 dev_priv->display.wm.pri_latency[3] = 0;
2854 dev_priv->display.wm.spr_latency[3] = 0;
2855 dev_priv->display.wm.cur_latency[3] = 0;
2856
2857 drm_dbg_kms(&dev_priv->drm,
2858 "LP3 watermarks disabled due to potential for lost interrupts\n");
2859 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2860 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2861 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2862}
2863
2864static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2865{
2866 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2867 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2868 else if (DISPLAY_VER(dev_priv) >= 6)
2869 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2870 else
2871 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2872
2873 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2874 sizeof(dev_priv->display.wm.pri_latency));
2875 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2876 sizeof(dev_priv->display.wm.pri_latency));
2877
2878 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2879 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2880
2881 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2882 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2883 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2884
2885 if (DISPLAY_VER(dev_priv) == 6) {
2886 snb_wm_latency_quirk(dev_priv);
2887 snb_wm_lp3_irq_quirk(dev_priv);
2888 }
2889}
2890
2891static bool ilk_validate_pipe_wm(struct drm_i915_private *dev_priv,
2892 struct intel_pipe_wm *pipe_wm)
2893{
2894 /* LP0 watermark maximums depend on this pipe alone */
2895 const struct intel_wm_config config = {
2896 .num_pipes_active = 1,
2897 .sprites_enabled = pipe_wm->sprites_enabled,
2898 .sprites_scaled = pipe_wm->sprites_scaled,
2899 };
2900 struct ilk_wm_maximums max;
2901
2902 /* LP0 watermarks always use 1/2 DDB partitioning */
2903 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2904
2905 /* At least LP0 must be valid */
2906 if (!ilk_validate_wm_level(dev_priv, 0, &max, &pipe_wm->wm[0])) {
2907 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2908 return false;
2909 }
2910
2911 return true;
2912}
2913
2914/* Compute new watermarks for the pipe */
2915static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2916 struct intel_crtc *crtc)
2917{
2918 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2919 struct intel_crtc_state *crtc_state =
2920 intel_atomic_get_new_crtc_state(state, crtc);
2921 struct intel_pipe_wm *pipe_wm;
2922 struct intel_plane *plane;
2923 const struct intel_plane_state *plane_state;
2924 const struct intel_plane_state *pristate = NULL;
2925 const struct intel_plane_state *sprstate = NULL;
2926 const struct intel_plane_state *curstate = NULL;
2927 struct ilk_wm_maximums max;
2928 int level, usable_level;
2929
2930 pipe_wm = &crtc_state->wm.ilk.optimal;
2931
2932 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2933 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2934 pristate = plane_state;
2935 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2936 sprstate = plane_state;
2937 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2938 curstate = plane_state;
2939 }
2940
2941 pipe_wm->pipe_enabled = crtc_state->hw.active;
2942 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2943 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2944
2945 usable_level = dev_priv->display.wm.num_levels - 1;
2946
2947 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2948 if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)
2949 usable_level = 1;
2950
2951 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2952 if (pipe_wm->sprites_scaled)
2953 usable_level = 0;
2954
2955 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2956 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2957 pristate, sprstate, curstate, &pipe_wm->wm[0]);
2958
2959 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2960 return -EINVAL;
2961
2962 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2963
2964 for (level = 1; level <= usable_level; level++) {
2965 struct intel_wm_level *wm = &pipe_wm->wm[level];
2966
2967 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2968 pristate, sprstate, curstate, wm);
2969
2970 /*
2971 * Disable any watermark level that exceeds the
2972 * register maximums since such watermarks are
2973 * always invalid.
2974 */
2975 if (!ilk_validate_wm_level(dev_priv, level, &max, wm)) {
2976 memset(wm, 0, sizeof(*wm));
2977 break;
2978 }
2979 }
2980
2981 return 0;
2982}
2983
2984/*
2985 * Build a set of 'intermediate' watermark values that satisfy both the old
2986 * state and the new state. These can be programmed to the hardware
2987 * immediately.
2988 */
2989static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2990 struct intel_crtc *crtc)
2991{
2992 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2993 struct intel_crtc_state *new_crtc_state =
2994 intel_atomic_get_new_crtc_state(state, crtc);
2995 const struct intel_crtc_state *old_crtc_state =
2996 intel_atomic_get_old_crtc_state(state, crtc);
2997 struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;
2998 const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;
2999 const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;
3000 int level;
3001
3002 /*
3003 * Start with the final, target watermarks, then combine with the
3004 * currently active watermarks to get values that are safe both before
3005 * and after the vblank.
3006 */
3007 *intermediate = *optimal;
3008 if (!new_crtc_state->hw.active ||
3009 intel_crtc_needs_modeset(new_crtc_state) ||
3010 state->skip_intermediate_wm)
3011 return 0;
3012
3013 intermediate->pipe_enabled |= active->pipe_enabled;
3014 intermediate->sprites_enabled |= active->sprites_enabled;
3015 intermediate->sprites_scaled |= active->sprites_scaled;
3016
3017 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3018 struct intel_wm_level *intermediate_wm = &intermediate->wm[level];
3019 const struct intel_wm_level *active_wm = &active->wm[level];
3020
3021 intermediate_wm->enable &= active_wm->enable;
3022 intermediate_wm->pri_val = max(intermediate_wm->pri_val,
3023 active_wm->pri_val);
3024 intermediate_wm->spr_val = max(intermediate_wm->spr_val,
3025 active_wm->spr_val);
3026 intermediate_wm->cur_val = max(intermediate_wm->cur_val,
3027 active_wm->cur_val);
3028 intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,
3029 active_wm->fbc_val);
3030 }
3031
3032 /*
3033 * We need to make sure that these merged watermark values are
3034 * actually a valid configuration themselves. If they're not,
3035 * there's no safe way to transition from the old state to
3036 * the new state, so we need to fail the atomic transaction.
3037 */
3038 if (!ilk_validate_pipe_wm(dev_priv, intermediate))
3039 return -EINVAL;
3040
3041 /*
3042 * If our intermediate WM are identical to the final WM, then we can
3043 * omit the post-vblank programming; only update if it's different.
3044 */
3045 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
3046 new_crtc_state->wm.need_postvbl_update = true;
3047
3048 return 0;
3049}
3050
3051static int ilk_compute_watermarks(struct intel_atomic_state *state,
3052 struct intel_crtc *crtc)
3053{
3054 int ret;
3055
3056 ret = ilk_compute_pipe_wm(state, crtc);
3057 if (ret)
3058 return ret;
3059
3060 ret = ilk_compute_intermediate_wm(state, crtc);
3061 if (ret)
3062 return ret;
3063
3064 return 0;
3065}
3066
3067/*
3068 * Merge the watermarks from all active pipes for a specific level.
3069 */
3070static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3071 int level,
3072 struct intel_wm_level *ret_wm)
3073{
3074 const struct intel_crtc *crtc;
3075
3076 ret_wm->enable = true;
3077
3078 for_each_intel_crtc(&dev_priv->drm, crtc) {
3079 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3080 const struct intel_wm_level *wm = &active->wm[level];
3081
3082 if (!active->pipe_enabled)
3083 continue;
3084
3085 /*
3086 * The watermark values may have been used in the past,
3087 * so we must maintain them in the registers for some
3088 * time even if the level is now disabled.
3089 */
3090 if (!wm->enable)
3091 ret_wm->enable = false;
3092
3093 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3094 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3095 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3096 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3097 }
3098}
3099
3100/*
3101 * Merge all low power watermarks for all active pipes.
3102 */
3103static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3104 const struct intel_wm_config *config,
3105 const struct ilk_wm_maximums *max,
3106 struct intel_pipe_wm *merged)
3107{
3108 int level, num_levels = dev_priv->display.wm.num_levels;
3109 int last_enabled_level = num_levels - 1;
3110
3111 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3112 if ((DISPLAY_VER(dev_priv) < 7 || IS_IVYBRIDGE(dev_priv)) &&
3113 config->num_pipes_active > 1)
3114 last_enabled_level = 0;
3115
3116 /* ILK: FBC WM must be disabled always */
3117 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3118
3119 /* merge each WM1+ level */
3120 for (level = 1; level < num_levels; level++) {
3121 struct intel_wm_level *wm = &merged->wm[level];
3122
3123 ilk_merge_wm_level(dev_priv, level, wm);
3124
3125 if (level > last_enabled_level)
3126 wm->enable = false;
3127 else if (!ilk_validate_wm_level(dev_priv, level, max, wm))
3128 /* make sure all following levels get disabled */
3129 last_enabled_level = level - 1;
3130
3131 /*
3132 * The spec says it is preferred to disable
3133 * FBC WMs instead of disabling a WM level.
3134 */
3135 if (wm->fbc_val > max->fbc) {
3136 if (wm->enable)
3137 merged->fbc_wm_enabled = false;
3138 wm->fbc_val = 0;
3139 }
3140 }
3141
3142 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3143 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3144 dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {
3145 for (level = 2; level < num_levels; level++) {
3146 struct intel_wm_level *wm = &merged->wm[level];
3147
3148 wm->enable = false;
3149 }
3150 }
3151}
3152
3153static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3154{
3155 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3156 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3157}
3158
3159/* The value we need to program into the WM_LPx latency field */
3160static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3161 int level)
3162{
3163 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3164 return 2 * level;
3165 else
3166 return dev_priv->display.wm.pri_latency[level];
3167}
3168
3169static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3170 const struct intel_pipe_wm *merged,
3171 enum intel_ddb_partitioning partitioning,
3172 struct ilk_wm_values *results)
3173{
3174 struct intel_crtc *crtc;
3175 int level, wm_lp;
3176
3177 results->enable_fbc_wm = merged->fbc_wm_enabled;
3178 results->partitioning = partitioning;
3179
3180 /* LP1+ register values */
3181 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3182 const struct intel_wm_level *r;
3183
3184 level = ilk_wm_lp_to_level(wm_lp, merged);
3185
3186 r = &merged->wm[level];
3187
3188 /*
3189 * Maintain the watermark values even if the level is
3190 * disabled. Doing otherwise could cause underruns.
3191 */
3192 results->wm_lp[wm_lp - 1] =
3193 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3194 WM_LP_PRIMARY(r->pri_val) |
3195 WM_LP_CURSOR(r->cur_val);
3196
3197 if (r->enable)
3198 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3199
3200 if (DISPLAY_VER(dev_priv) >= 8)
3201 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3202 else
3203 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3204
3205 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3206
3207 /*
3208 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3209 * level is disabled. Doing otherwise could cause underruns.
3210 */
3211 if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {
3212 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3213 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3214 }
3215 }
3216
3217 /* LP0 register values */
3218 for_each_intel_crtc(&dev_priv->drm, crtc) {
3219 enum pipe pipe = crtc->pipe;
3220 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3221 const struct intel_wm_level *r = &pipe_wm->wm[0];
3222
3223 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3224 continue;
3225
3226 results->wm_pipe[pipe] =
3227 WM0_PIPE_PRIMARY(r->pri_val) |
3228 WM0_PIPE_SPRITE(r->spr_val) |
3229 WM0_PIPE_CURSOR(r->cur_val);
3230 }
3231}
3232
3233/*
3234 * Find the result with the highest level enabled. Check for enable_fbc_wm in
3235 * case both are at the same level. Prefer r1 in case they're the same.
3236 */
3237static struct intel_pipe_wm *
3238ilk_find_best_result(struct drm_i915_private *dev_priv,
3239 struct intel_pipe_wm *r1,
3240 struct intel_pipe_wm *r2)
3241{
3242 int level, level1 = 0, level2 = 0;
3243
3244 for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3245 if (r1->wm[level].enable)
3246 level1 = level;
3247 if (r2->wm[level].enable)
3248 level2 = level;
3249 }
3250
3251 if (level1 == level2) {
3252 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3253 return r2;
3254 else
3255 return r1;
3256 } else if (level1 > level2) {
3257 return r1;
3258 } else {
3259 return r2;
3260 }
3261}
3262
3263/* dirty bits used to track which watermarks need changes */
3264#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3265#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3266#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3267#define WM_DIRTY_FBC (1 << 24)
3268#define WM_DIRTY_DDB (1 << 25)
3269
3270static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3271 const struct ilk_wm_values *old,
3272 const struct ilk_wm_values *new)
3273{
3274 unsigned int dirty = 0;
3275 enum pipe pipe;
3276 int wm_lp;
3277
3278 for_each_pipe(dev_priv, pipe) {
3279 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3280 dirty |= WM_DIRTY_PIPE(pipe);
3281 /* Must disable LP1+ watermarks too */
3282 dirty |= WM_DIRTY_LP_ALL;
3283 }
3284 }
3285
3286 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3287 dirty |= WM_DIRTY_FBC;
3288 /* Must disable LP1+ watermarks too */
3289 dirty |= WM_DIRTY_LP_ALL;
3290 }
3291
3292 if (old->partitioning != new->partitioning) {
3293 dirty |= WM_DIRTY_DDB;
3294 /* Must disable LP1+ watermarks too */
3295 dirty |= WM_DIRTY_LP_ALL;
3296 }
3297
3298 /* LP1+ watermarks already deemed dirty, no need to continue */
3299 if (dirty & WM_DIRTY_LP_ALL)
3300 return dirty;
3301
3302 /* Find the lowest numbered LP1+ watermark in need of an update... */
3303 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3304 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3305 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3306 break;
3307 }
3308
3309 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3310 for (; wm_lp <= 3; wm_lp++)
3311 dirty |= WM_DIRTY_LP(wm_lp);
3312
3313 return dirty;
3314}
3315
3316static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3317 unsigned int dirty)
3318{
3319 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3320 bool changed = false;
3321
3322 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3323 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3324 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3325 changed = true;
3326 }
3327 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3328 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3329 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3330 changed = true;
3331 }
3332 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3333 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3334 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3335 changed = true;
3336 }
3337
3338 /*
3339 * Don't touch WM_LP_SPRITE_ENABLE here.
3340 * Doing so could cause underruns.
3341 */
3342
3343 return changed;
3344}
3345
3346/*
3347 * The spec says we shouldn't write when we don't need, because every write
3348 * causes WMs to be re-evaluated, expending some power.
3349 */
3350static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3351 struct ilk_wm_values *results)
3352{
3353 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3354 unsigned int dirty;
3355
3356 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3357 if (!dirty)
3358 return;
3359
3360 _ilk_disable_lp_wm(dev_priv, dirty);
3361
3362 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3363 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3364 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3365 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3366 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3367 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3368
3369 if (dirty & WM_DIRTY_DDB) {
3370 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3371 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3372 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3373 WM_MISC_DATA_PARTITION_5_6);
3374 else
3375 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3376 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3377 DISP_DATA_PARTITION_5_6);
3378 }
3379
3380 if (dirty & WM_DIRTY_FBC)
3381 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3382 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3383
3384 if (dirty & WM_DIRTY_LP(1) &&
3385 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3386 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3387
3388 if (DISPLAY_VER(dev_priv) >= 7) {
3389 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3390 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3391 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3392 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3393 }
3394
3395 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3396 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3397 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3398 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3399 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3400 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3401
3402 dev_priv->display.wm.hw = *results;
3403}
3404
3405bool ilk_disable_cxsr(struct drm_i915_private *dev_priv)
3406{
3407 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3408}
3409
3410static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3411 struct intel_wm_config *config)
3412{
3413 struct intel_crtc *crtc;
3414
3415 /* Compute the currently _active_ config */
3416 for_each_intel_crtc(&dev_priv->drm, crtc) {
3417 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3418
3419 if (!wm->pipe_enabled)
3420 continue;
3421
3422 config->sprites_enabled |= wm->sprites_enabled;
3423 config->sprites_scaled |= wm->sprites_scaled;
3424 config->num_pipes_active++;
3425 }
3426}
3427
3428static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3429{
3430 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3431 struct ilk_wm_maximums max;
3432 struct intel_wm_config config = {};
3433 struct ilk_wm_values results = {};
3434 enum intel_ddb_partitioning partitioning;
3435
3436 ilk_compute_wm_config(dev_priv, &config);
3437
3438 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3439 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3440
3441 /* 5/6 split only in single pipe config on IVB+ */
3442 if (DISPLAY_VER(dev_priv) >= 7 &&
3443 config.num_pipes_active == 1 && config.sprites_enabled) {
3444 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3445 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3446
3447 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3448 } else {
3449 best_lp_wm = &lp_wm_1_2;
3450 }
3451
3452 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3453 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3454
3455 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3456
3457 ilk_write_wm_values(dev_priv, &results);
3458}
3459
3460static void ilk_initial_watermarks(struct intel_atomic_state *state,
3461 struct intel_crtc *crtc)
3462{
3463 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3464 const struct intel_crtc_state *crtc_state =
3465 intel_atomic_get_new_crtc_state(state, crtc);
3466
3467 mutex_lock(&dev_priv->display.wm.wm_mutex);
3468 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3469 ilk_program_watermarks(dev_priv);
3470 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3471}
3472
3473static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3474 struct intel_crtc *crtc)
3475{
3476 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3477 const struct intel_crtc_state *crtc_state =
3478 intel_atomic_get_new_crtc_state(state, crtc);
3479
3480 if (!crtc_state->wm.need_postvbl_update)
3481 return;
3482
3483 mutex_lock(&dev_priv->display.wm.wm_mutex);
3484 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3485 ilk_program_watermarks(dev_priv);
3486 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3487}
3488
3489static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3490{
3491 struct drm_device *dev = crtc->base.dev;
3492 struct drm_i915_private *dev_priv = to_i915(dev);
3493 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3494 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3495 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3496 enum pipe pipe = crtc->pipe;
3497
3498 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3499
3500 memset(active, 0, sizeof(*active));
3501
3502 active->pipe_enabled = crtc->active;
3503
3504 if (active->pipe_enabled) {
3505 u32 tmp = hw->wm_pipe[pipe];
3506
3507 /*
3508 * For active pipes LP0 watermark is marked as
3509 * enabled, and LP1+ watermaks as disabled since
3510 * we can't really reverse compute them in case
3511 * multiple pipes are active.
3512 */
3513 active->wm[0].enable = true;
3514 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3515 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3516 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3517 } else {
3518 int level;
3519
3520 /*
3521 * For inactive pipes, all watermark levels
3522 * should be marked as enabled but zeroed,
3523 * which is what we'd compute them to.
3524 */
3525 for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3526 active->wm[level].enable = true;
3527 }
3528
3529 crtc->wm.active.ilk = *active;
3530}
3531
3532static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3533{
3534 struct drm_plane *plane;
3535 struct intel_crtc *crtc;
3536
3537 for_each_intel_crtc(state->dev, crtc) {
3538 struct intel_crtc_state *crtc_state;
3539
3540 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3541 if (IS_ERR(crtc_state))
3542 return PTR_ERR(crtc_state);
3543
3544 if (crtc_state->hw.active) {
3545 /*
3546 * Preserve the inherited flag to avoid
3547 * taking the full modeset path.
3548 */
3549 crtc_state->inherited = true;
3550 }
3551 }
3552
3553 drm_for_each_plane(plane, state->dev) {
3554 struct drm_plane_state *plane_state;
3555
3556 plane_state = drm_atomic_get_plane_state(state, plane);
3557 if (IS_ERR(plane_state))
3558 return PTR_ERR(plane_state);
3559 }
3560
3561 return 0;
3562}
3563
3564/*
3565 * Calculate what we think the watermarks should be for the state we've read
3566 * out of the hardware and then immediately program those watermarks so that
3567 * we ensure the hardware settings match our internal state.
3568 *
3569 * We can calculate what we think WM's should be by creating a duplicate of the
3570 * current state (which was constructed during hardware readout) and running it
3571 * through the atomic check code to calculate new watermark values in the
3572 * state object.
3573 */
3574void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3575{
3576 struct drm_atomic_state *state;
3577 struct intel_atomic_state *intel_state;
3578 struct intel_crtc *crtc;
3579 struct intel_crtc_state *crtc_state;
3580 struct drm_modeset_acquire_ctx ctx;
3581 int ret;
3582 int i;
3583
3584 /* Only supported on platforms that use atomic watermark design */
3585 if (!dev_priv->display.funcs.wm->optimize_watermarks)
3586 return;
3587
3588 if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3589 return;
3590
3591 state = drm_atomic_state_alloc(&dev_priv->drm);
3592 if (drm_WARN_ON(&dev_priv->drm, !state))
3593 return;
3594
3595 intel_state = to_intel_atomic_state(state);
3596
3597 drm_modeset_acquire_init(&ctx, 0);
3598
3599 state->acquire_ctx = &ctx;
3600 to_intel_atomic_state(state)->internal = true;
3601
3602retry:
3603 /*
3604 * Hardware readout is the only time we don't want to calculate
3605 * intermediate watermarks (since we don't trust the current
3606 * watermarks).
3607 */
3608 if (!HAS_GMCH(dev_priv))
3609 intel_state->skip_intermediate_wm = true;
3610
3611 ret = ilk_sanitize_watermarks_add_affected(state);
3612 if (ret)
3613 goto fail;
3614
3615 ret = intel_atomic_check(&dev_priv->drm, state);
3616 if (ret)
3617 goto fail;
3618
3619 /* Write calculated watermark values back */
3620 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3621 crtc_state->wm.need_postvbl_update = true;
3622 intel_optimize_watermarks(intel_state, crtc);
3623
3624 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3625 }
3626
3627fail:
3628 if (ret == -EDEADLK) {
3629 drm_atomic_state_clear(state);
3630 drm_modeset_backoff(&ctx);
3631 goto retry;
3632 }
3633
3634 /*
3635 * If we fail here, it means that the hardware appears to be
3636 * programmed in a way that shouldn't be possible, given our
3637 * understanding of watermark requirements. This might mean a
3638 * mistake in the hardware readout code or a mistake in the
3639 * watermark calculations for a given platform. Raise a WARN
3640 * so that this is noticeable.
3641 *
3642 * If this actually happens, we'll have to just leave the
3643 * BIOS-programmed watermarks untouched and hope for the best.
3644 */
3645 drm_WARN(&dev_priv->drm, ret,
3646 "Could not determine valid watermarks for inherited state\n");
3647
3648 drm_atomic_state_put(state);
3649
3650 drm_modeset_drop_locks(&ctx);
3651 drm_modeset_acquire_fini(&ctx);
3652}
3653
3654#define _FW_WM(value, plane) \
3655 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3656#define _FW_WM_VLV(value, plane) \
3657 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3658
3659static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3660 struct g4x_wm_values *wm)
3661{
3662 u32 tmp;
3663
3664 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3665 wm->sr.plane = _FW_WM(tmp, SR);
3666 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3667 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3668 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3669
3670 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3671 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3672 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3673 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3674 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3675 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3676 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3677
3678 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3679 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3680 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3681 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3682 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3683}
3684
3685static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3686 struct vlv_wm_values *wm)
3687{
3688 enum pipe pipe;
3689 u32 tmp;
3690
3691 for_each_pipe(dev_priv, pipe) {
3692 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3693
3694 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3695 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3696 wm->ddl[pipe].plane[PLANE_CURSOR] =
3697 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3698 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3699 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3700 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3701 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3702 }
3703
3704 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3705 wm->sr.plane = _FW_WM(tmp, SR);
3706 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3707 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3708 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3709
3710 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3711 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3712 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3713 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3714
3715 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3716 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3717
3718 if (IS_CHERRYVIEW(dev_priv)) {
3719 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3720 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3721 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3722
3723 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3724 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3725 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3726
3727 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3728 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3729 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3730
3731 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3732 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3733 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3734 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3735 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3736 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3737 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3738 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3739 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3740 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3741 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3742 } else {
3743 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3744 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3745 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3746
3747 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3748 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3749 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3750 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3751 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3752 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3753 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3754 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3755 }
3756}
3757
3758#undef _FW_WM
3759#undef _FW_WM_VLV
3760
3761static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3762{
3763 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3764 struct intel_crtc *crtc;
3765
3766 g4x_read_wm_values(dev_priv, wm);
3767
3768 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3769
3770 for_each_intel_crtc(&dev_priv->drm, crtc) {
3771 struct intel_crtc_state *crtc_state =
3772 to_intel_crtc_state(crtc->base.state);
3773 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3774 struct g4x_pipe_wm *raw;
3775 enum pipe pipe = crtc->pipe;
3776 enum plane_id plane_id;
3777 int level, max_level;
3778
3779 active->cxsr = wm->cxsr;
3780 active->hpll_en = wm->hpll_en;
3781 active->fbc_en = wm->fbc_en;
3782
3783 active->sr = wm->sr;
3784 active->hpll = wm->hpll;
3785
3786 for_each_plane_id_on_crtc(crtc, plane_id) {
3787 active->wm.plane[plane_id] =
3788 wm->pipe[pipe].plane[plane_id];
3789 }
3790
3791 if (wm->cxsr && wm->hpll_en)
3792 max_level = G4X_WM_LEVEL_HPLL;
3793 else if (wm->cxsr)
3794 max_level = G4X_WM_LEVEL_SR;
3795 else
3796 max_level = G4X_WM_LEVEL_NORMAL;
3797
3798 level = G4X_WM_LEVEL_NORMAL;
3799 raw = &crtc_state->wm.g4x.raw[level];
3800 for_each_plane_id_on_crtc(crtc, plane_id)
3801 raw->plane[plane_id] = active->wm.plane[plane_id];
3802
3803 level = G4X_WM_LEVEL_SR;
3804 if (level > max_level)
3805 goto out;
3806
3807 raw = &crtc_state->wm.g4x.raw[level];
3808 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3809 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3810 raw->plane[PLANE_SPRITE0] = 0;
3811 raw->fbc = active->sr.fbc;
3812
3813 level = G4X_WM_LEVEL_HPLL;
3814 if (level > max_level)
3815 goto out;
3816
3817 raw = &crtc_state->wm.g4x.raw[level];
3818 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3819 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3820 raw->plane[PLANE_SPRITE0] = 0;
3821 raw->fbc = active->hpll.fbc;
3822
3823 level++;
3824 out:
3825 for_each_plane_id_on_crtc(crtc, plane_id)
3826 g4x_raw_plane_wm_set(crtc_state, level,
3827 plane_id, USHRT_MAX);
3828 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3829
3830 g4x_invalidate_wms(crtc, active, level);
3831
3832 crtc_state->wm.g4x.optimal = *active;
3833 crtc_state->wm.g4x.intermediate = *active;
3834
3835 drm_dbg_kms(&dev_priv->drm,
3836 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3837 pipe_name(pipe),
3838 wm->pipe[pipe].plane[PLANE_PRIMARY],
3839 wm->pipe[pipe].plane[PLANE_CURSOR],
3840 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3841 }
3842
3843 drm_dbg_kms(&dev_priv->drm,
3844 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3845 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3846 drm_dbg_kms(&dev_priv->drm,
3847 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3848 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3849 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3850 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3851 str_yes_no(wm->fbc_en));
3852}
3853
3854static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3855{
3856 struct intel_display *display = &dev_priv->display;
3857 struct intel_plane *plane;
3858 struct intel_crtc *crtc;
3859
3860 mutex_lock(&dev_priv->display.wm.wm_mutex);
3861
3862 for_each_intel_plane(&dev_priv->drm, plane) {
3863 struct intel_crtc *crtc =
3864 intel_crtc_for_pipe(display, plane->pipe);
3865 struct intel_crtc_state *crtc_state =
3866 to_intel_crtc_state(crtc->base.state);
3867 struct intel_plane_state *plane_state =
3868 to_intel_plane_state(plane->base.state);
3869 enum plane_id plane_id = plane->id;
3870 int level;
3871
3872 if (plane_state->uapi.visible)
3873 continue;
3874
3875 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3876 struct g4x_pipe_wm *raw =
3877 &crtc_state->wm.g4x.raw[level];
3878
3879 raw->plane[plane_id] = 0;
3880
3881 if (plane_id == PLANE_PRIMARY)
3882 raw->fbc = 0;
3883 }
3884 }
3885
3886 for_each_intel_crtc(&dev_priv->drm, crtc) {
3887 struct intel_crtc_state *crtc_state =
3888 to_intel_crtc_state(crtc->base.state);
3889 int ret;
3890
3891 ret = _g4x_compute_pipe_wm(crtc_state);
3892 drm_WARN_ON(&dev_priv->drm, ret);
3893
3894 crtc_state->wm.g4x.intermediate =
3895 crtc_state->wm.g4x.optimal;
3896 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3897 }
3898
3899 g4x_program_watermarks(dev_priv);
3900
3901 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3902}
3903
3904static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3905{
3906 g4x_wm_get_hw_state(i915);
3907 g4x_wm_sanitize(i915);
3908}
3909
3910static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3911{
3912 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3913 struct intel_crtc *crtc;
3914 u32 val;
3915
3916 vlv_read_wm_values(dev_priv, wm);
3917
3918 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3919 wm->level = VLV_WM_LEVEL_PM2;
3920
3921 if (IS_CHERRYVIEW(dev_priv)) {
3922 vlv_punit_get(dev_priv);
3923
3924 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3925 if (val & DSP_MAXFIFO_PM5_ENABLE)
3926 wm->level = VLV_WM_LEVEL_PM5;
3927
3928 /*
3929 * If DDR DVFS is disabled in the BIOS, Punit
3930 * will never ack the request. So if that happens
3931 * assume we don't have to enable/disable DDR DVFS
3932 * dynamically. To test that just set the REQ_ACK
3933 * bit to poke the Punit, but don't change the
3934 * HIGH/LOW bits so that we don't actually change
3935 * the current state.
3936 */
3937 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3938 val |= FORCE_DDR_FREQ_REQ_ACK;
3939 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3940
3941 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3942 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3943 drm_dbg_kms(&dev_priv->drm,
3944 "Punit not acking DDR DVFS request, "
3945 "assuming DDR DVFS is disabled\n");
3946 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3947 } else {
3948 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3949 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3950 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3951 }
3952
3953 vlv_punit_put(dev_priv);
3954 }
3955
3956 for_each_intel_crtc(&dev_priv->drm, crtc) {
3957 struct intel_crtc_state *crtc_state =
3958 to_intel_crtc_state(crtc->base.state);
3959 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3960 const struct vlv_fifo_state *fifo_state =
3961 &crtc_state->wm.vlv.fifo_state;
3962 enum pipe pipe = crtc->pipe;
3963 enum plane_id plane_id;
3964 int level;
3965
3966 vlv_get_fifo_size(crtc_state);
3967
3968 active->num_levels = wm->level + 1;
3969 active->cxsr = wm->cxsr;
3970
3971 for (level = 0; level < active->num_levels; level++) {
3972 struct g4x_pipe_wm *raw =
3973 &crtc_state->wm.vlv.raw[level];
3974
3975 active->sr[level].plane = wm->sr.plane;
3976 active->sr[level].cursor = wm->sr.cursor;
3977
3978 for_each_plane_id_on_crtc(crtc, plane_id) {
3979 active->wm[level].plane[plane_id] =
3980 wm->pipe[pipe].plane[plane_id];
3981
3982 raw->plane[plane_id] =
3983 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3984 fifo_state->plane[plane_id]);
3985 }
3986 }
3987
3988 for_each_plane_id_on_crtc(crtc, plane_id)
3989 vlv_raw_plane_wm_set(crtc_state, level,
3990 plane_id, USHRT_MAX);
3991 vlv_invalidate_wms(crtc, active, level);
3992
3993 crtc_state->wm.vlv.optimal = *active;
3994 crtc_state->wm.vlv.intermediate = *active;
3995
3996 drm_dbg_kms(&dev_priv->drm,
3997 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3998 pipe_name(pipe),
3999 wm->pipe[pipe].plane[PLANE_PRIMARY],
4000 wm->pipe[pipe].plane[PLANE_CURSOR],
4001 wm->pipe[pipe].plane[PLANE_SPRITE0],
4002 wm->pipe[pipe].plane[PLANE_SPRITE1]);
4003 }
4004
4005 drm_dbg_kms(&dev_priv->drm,
4006 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4007 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4008}
4009
4010static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
4011{
4012 struct intel_display *display = &dev_priv->display;
4013 struct intel_plane *plane;
4014 struct intel_crtc *crtc;
4015
4016 mutex_lock(&dev_priv->display.wm.wm_mutex);
4017
4018 for_each_intel_plane(&dev_priv->drm, plane) {
4019 struct intel_crtc *crtc =
4020 intel_crtc_for_pipe(display, plane->pipe);
4021 struct intel_crtc_state *crtc_state =
4022 to_intel_crtc_state(crtc->base.state);
4023 struct intel_plane_state *plane_state =
4024 to_intel_plane_state(plane->base.state);
4025 enum plane_id plane_id = plane->id;
4026 int level;
4027
4028 if (plane_state->uapi.visible)
4029 continue;
4030
4031 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
4032 struct g4x_pipe_wm *raw =
4033 &crtc_state->wm.vlv.raw[level];
4034
4035 raw->plane[plane_id] = 0;
4036 }
4037 }
4038
4039 for_each_intel_crtc(&dev_priv->drm, crtc) {
4040 struct intel_crtc_state *crtc_state =
4041 to_intel_crtc_state(crtc->base.state);
4042 int ret;
4043
4044 ret = _vlv_compute_pipe_wm(crtc_state);
4045 drm_WARN_ON(&dev_priv->drm, ret);
4046
4047 crtc_state->wm.vlv.intermediate =
4048 crtc_state->wm.vlv.optimal;
4049 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4050 }
4051
4052 vlv_program_watermarks(dev_priv);
4053
4054 mutex_unlock(&dev_priv->display.wm.wm_mutex);
4055}
4056
4057static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
4058{
4059 vlv_wm_get_hw_state(i915);
4060 vlv_wm_sanitize(i915);
4061}
4062
4063/*
4064 * FIXME should probably kill this and improve
4065 * the real watermark readout/sanitation instead
4066 */
4067static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4068{
4069 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4070 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4071 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4072
4073 /*
4074 * Don't touch WM_LP_SPRITE_ENABLE here.
4075 * Doing so could cause underruns.
4076 */
4077}
4078
4079static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4080{
4081 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4082 struct intel_crtc *crtc;
4083
4084 ilk_init_lp_watermarks(dev_priv);
4085
4086 for_each_intel_crtc(&dev_priv->drm, crtc)
4087 ilk_pipe_wm_get_hw_state(crtc);
4088
4089 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4090 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4091 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4092
4093 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4094 if (DISPLAY_VER(dev_priv) >= 7) {
4095 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4096 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4097 }
4098
4099 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4100 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
4101 WM_MISC_DATA_PARTITION_5_6) ?
4102 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4103 else if (IS_IVYBRIDGE(dev_priv))
4104 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
4105 DISP_DATA_PARTITION_5_6) ?
4106 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4107
4108 hw->enable_fbc_wm =
4109 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4110}
4111
4112static const struct intel_wm_funcs ilk_wm_funcs = {
4113 .compute_watermarks = ilk_compute_watermarks,
4114 .initial_watermarks = ilk_initial_watermarks,
4115 .optimize_watermarks = ilk_optimize_watermarks,
4116 .get_hw_state = ilk_wm_get_hw_state,
4117};
4118
4119static const struct intel_wm_funcs vlv_wm_funcs = {
4120 .compute_watermarks = vlv_compute_watermarks,
4121 .initial_watermarks = vlv_initial_watermarks,
4122 .optimize_watermarks = vlv_optimize_watermarks,
4123 .atomic_update_watermarks = vlv_atomic_update_fifo,
4124 .get_hw_state = vlv_wm_get_hw_state_and_sanitize,
4125};
4126
4127static const struct intel_wm_funcs g4x_wm_funcs = {
4128 .compute_watermarks = g4x_compute_watermarks,
4129 .initial_watermarks = g4x_initial_watermarks,
4130 .optimize_watermarks = g4x_optimize_watermarks,
4131 .get_hw_state = g4x_wm_get_hw_state_and_sanitize,
4132};
4133
4134static const struct intel_wm_funcs pnv_wm_funcs = {
4135 .compute_watermarks = i9xx_compute_watermarks,
4136 .update_wm = pnv_update_wm,
4137};
4138
4139static const struct intel_wm_funcs i965_wm_funcs = {
4140 .compute_watermarks = i9xx_compute_watermarks,
4141 .update_wm = i965_update_wm,
4142};
4143
4144static const struct intel_wm_funcs i9xx_wm_funcs = {
4145 .compute_watermarks = i9xx_compute_watermarks,
4146 .update_wm = i9xx_update_wm,
4147};
4148
4149static const struct intel_wm_funcs i845_wm_funcs = {
4150 .compute_watermarks = i9xx_compute_watermarks,
4151 .update_wm = i845_update_wm,
4152};
4153
4154static const struct intel_wm_funcs nop_funcs = {
4155};
4156
4157void i9xx_wm_init(struct drm_i915_private *dev_priv)
4158{
4159 /* For FIFO watermark updates */
4160 if (HAS_PCH_SPLIT(dev_priv)) {
4161 ilk_setup_wm_latency(dev_priv);
4162 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4163 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4164 vlv_setup_wm_latency(dev_priv);
4165 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4166 } else if (IS_G4X(dev_priv)) {
4167 g4x_setup_wm_latency(dev_priv);
4168 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4169 } else if (IS_PINEVIEW(dev_priv)) {
4170 if (!pnv_get_cxsr_latency(dev_priv)) {
4171 drm_info(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
4172 /* Disable CxSR and never update its watermark again */
4173 intel_set_memory_cxsr(dev_priv, false);
4174 dev_priv->display.funcs.wm = &nop_funcs;
4175 } else {
4176 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4177 }
4178 } else if (DISPLAY_VER(dev_priv) == 4) {
4179 dev_priv->display.funcs.wm = &i965_wm_funcs;
4180 } else if (DISPLAY_VER(dev_priv) == 3) {
4181 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4182 } else if (DISPLAY_VER(dev_priv) == 2) {
4183 if (INTEL_NUM_PIPES(dev_priv) == 1)
4184 dev_priv->display.funcs.wm = &i845_wm_funcs;
4185 else
4186 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4187 } else {
4188 drm_err(&dev_priv->drm,
4189 "unexpected fall-through in %s\n", __func__);
4190 dev_priv->display.funcs.wm = &nop_funcs;
4191 }
4192}
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "i915_reg.h"
8#include "i9xx_wm.h"
9#include "intel_atomic.h"
10#include "intel_display.h"
11#include "intel_display_trace.h"
12#include "intel_mchbar_regs.h"
13#include "intel_wm.h"
14#include "skl_watermark.h"
15#include "vlv_sideband.h"
16
17/* used in computing the new watermarks state */
18struct intel_wm_config {
19 unsigned int num_pipes_active;
20 bool sprites_enabled;
21 bool sprites_scaled;
22};
23
24struct cxsr_latency {
25 bool is_desktop : 1;
26 bool is_ddr3 : 1;
27 u16 fsb_freq;
28 u16 mem_freq;
29 u16 display_sr;
30 u16 display_hpll_disable;
31 u16 cursor_sr;
32 u16 cursor_hpll_disable;
33};
34
35static const struct cxsr_latency cxsr_latency_table[] = {
36 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
37 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
38 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
39 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
40 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
41
42 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
43 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
44 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
45 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
46 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
47
48 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
49 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
50 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
51 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
52 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
53
54 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
55 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
56 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
57 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
58 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
59
60 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
61 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
62 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
63 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
64 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
65
66 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
67 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
68 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
69 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
70 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
71};
72
73static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
74 bool is_ddr3,
75 int fsb,
76 int mem)
77{
78 const struct cxsr_latency *latency;
79 int i;
80
81 if (fsb == 0 || mem == 0)
82 return NULL;
83
84 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
85 latency = &cxsr_latency_table[i];
86 if (is_desktop == latency->is_desktop &&
87 is_ddr3 == latency->is_ddr3 &&
88 fsb == latency->fsb_freq && mem == latency->mem_freq)
89 return latency;
90 }
91
92 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
93
94 return NULL;
95}
96
97static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
98{
99 u32 val;
100
101 vlv_punit_get(dev_priv);
102
103 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
104 if (enable)
105 val &= ~FORCE_DDR_HIGH_FREQ;
106 else
107 val |= FORCE_DDR_HIGH_FREQ;
108 val &= ~FORCE_DDR_LOW_FREQ;
109 val |= FORCE_DDR_FREQ_REQ_ACK;
110 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
111
112 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
113 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
114 drm_err(&dev_priv->drm,
115 "timed out waiting for Punit DDR DVFS request\n");
116
117 vlv_punit_put(dev_priv);
118}
119
120static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
121{
122 u32 val;
123
124 vlv_punit_get(dev_priv);
125
126 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
127 if (enable)
128 val |= DSP_MAXFIFO_PM5_ENABLE;
129 else
130 val &= ~DSP_MAXFIFO_PM5_ENABLE;
131 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
132
133 vlv_punit_put(dev_priv);
134}
135
136#define FW_WM(value, plane) \
137 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
138
139static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
140{
141 bool was_enabled;
142 u32 val;
143
144 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
145 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
146 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
147 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
148 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
149 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
150 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
151 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
152 } else if (IS_PINEVIEW(dev_priv)) {
153 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
154 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
155 if (enable)
156 val |= PINEVIEW_SELF_REFRESH_EN;
157 else
158 val &= ~PINEVIEW_SELF_REFRESH_EN;
159 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
160 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
161 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
162 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
163 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
164 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
165 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
166 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
167 } else if (IS_I915GM(dev_priv)) {
168 /*
169 * FIXME can't find a bit like this for 915G, and
170 * yet it does have the related watermark in
171 * FW_BLC_SELF. What's going on?
172 */
173 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
174 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
175 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
176 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
177 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
178 } else {
179 return false;
180 }
181
182 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
183
184 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
185 str_enabled_disabled(enable),
186 str_enabled_disabled(was_enabled));
187
188 return was_enabled;
189}
190
191/**
192 * intel_set_memory_cxsr - Configure CxSR state
193 * @dev_priv: i915 device
194 * @enable: Allow vs. disallow CxSR
195 *
196 * Allow or disallow the system to enter a special CxSR
197 * (C-state self refresh) state. What typically happens in CxSR mode
198 * is that several display FIFOs may get combined into a single larger
199 * FIFO for a particular plane (so called max FIFO mode) to allow the
200 * system to defer memory fetches longer, and the memory will enter
201 * self refresh.
202 *
203 * Note that enabling CxSR does not guarantee that the system enter
204 * this special mode, nor does it guarantee that the system stays
205 * in that mode once entered. So this just allows/disallows the system
206 * to autonomously utilize the CxSR mode. Other factors such as core
207 * C-states will affect when/if the system actually enters/exits the
208 * CxSR mode.
209 *
210 * Note that on VLV/CHV this actually only controls the max FIFO mode,
211 * and the system is free to enter/exit memory self refresh at any time
212 * even when the use of CxSR has been disallowed.
213 *
214 * While the system is actually in the CxSR/max FIFO mode, some plane
215 * control registers will not get latched on vblank. Thus in order to
216 * guarantee the system will respond to changes in the plane registers
217 * we must always disallow CxSR prior to making changes to those registers.
218 * Unfortunately the system will re-evaluate the CxSR conditions at
219 * frame start which happens after vblank start (which is when the plane
220 * registers would get latched), so we can't proceed with the plane update
221 * during the same frame where we disallowed CxSR.
222 *
223 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
224 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
225 * the hardware w.r.t. HPLL SR when writing to plane registers.
226 * Disallowing just CxSR is sufficient.
227 */
228bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
229{
230 bool ret;
231
232 mutex_lock(&dev_priv->display.wm.wm_mutex);
233 ret = _intel_set_memory_cxsr(dev_priv, enable);
234 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
235 dev_priv->display.wm.vlv.cxsr = enable;
236 else if (IS_G4X(dev_priv))
237 dev_priv->display.wm.g4x.cxsr = enable;
238 mutex_unlock(&dev_priv->display.wm.wm_mutex);
239
240 return ret;
241}
242
243/*
244 * Latency for FIFO fetches is dependent on several factors:
245 * - memory configuration (speed, channels)
246 * - chipset
247 * - current MCH state
248 * It can be fairly high in some situations, so here we assume a fairly
249 * pessimal value. It's a tradeoff between extra memory fetches (if we
250 * set this value too high, the FIFO will fetch frequently to stay full)
251 * and power consumption (set it too low to save power and we might see
252 * FIFO underruns and display "flicker").
253 *
254 * A value of 5us seems to be a good balance; safe for very low end
255 * platforms but not overly aggressive on lower latency configs.
256 */
257static const int pessimal_latency_ns = 5000;
258
259#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
260 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
261
262static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
263{
264 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
265 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
266 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
267 enum pipe pipe = crtc->pipe;
268 int sprite0_start, sprite1_start;
269 u32 dsparb, dsparb2, dsparb3;
270
271 switch (pipe) {
272 case PIPE_A:
273 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
274 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
275 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
276 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
277 break;
278 case PIPE_B:
279 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
280 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
281 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
282 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
283 break;
284 case PIPE_C:
285 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
286 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
287 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
288 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
289 break;
290 default:
291 MISSING_CASE(pipe);
292 return;
293 }
294
295 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
296 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
297 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
298 fifo_state->plane[PLANE_CURSOR] = 63;
299}
300
301static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
302 enum i9xx_plane_id i9xx_plane)
303{
304 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
305 int size;
306
307 size = dsparb & 0x7f;
308 if (i9xx_plane == PLANE_B)
309 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
310
311 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
312 dsparb, plane_name(i9xx_plane), size);
313
314 return size;
315}
316
317static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
318 enum i9xx_plane_id i9xx_plane)
319{
320 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
321 int size;
322
323 size = dsparb & 0x1ff;
324 if (i9xx_plane == PLANE_B)
325 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
326 size >>= 1; /* Convert to cachelines */
327
328 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
329 dsparb, plane_name(i9xx_plane), size);
330
331 return size;
332}
333
334static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
335 enum i9xx_plane_id i9xx_plane)
336{
337 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
338 int size;
339
340 size = dsparb & 0x7f;
341 size >>= 2; /* Convert to cachelines */
342
343 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
344 dsparb, plane_name(i9xx_plane), size);
345
346 return size;
347}
348
349/* Pineview has different values for various configs */
350static const struct intel_watermark_params pnv_display_wm = {
351 .fifo_size = PINEVIEW_DISPLAY_FIFO,
352 .max_wm = PINEVIEW_MAX_WM,
353 .default_wm = PINEVIEW_DFT_WM,
354 .guard_size = PINEVIEW_GUARD_WM,
355 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
356};
357
358static const struct intel_watermark_params pnv_display_hplloff_wm = {
359 .fifo_size = PINEVIEW_DISPLAY_FIFO,
360 .max_wm = PINEVIEW_MAX_WM,
361 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
362 .guard_size = PINEVIEW_GUARD_WM,
363 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
364};
365
366static const struct intel_watermark_params pnv_cursor_wm = {
367 .fifo_size = PINEVIEW_CURSOR_FIFO,
368 .max_wm = PINEVIEW_CURSOR_MAX_WM,
369 .default_wm = PINEVIEW_CURSOR_DFT_WM,
370 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
371 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
372};
373
374static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
375 .fifo_size = PINEVIEW_CURSOR_FIFO,
376 .max_wm = PINEVIEW_CURSOR_MAX_WM,
377 .default_wm = PINEVIEW_CURSOR_DFT_WM,
378 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
379 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
380};
381
382static const struct intel_watermark_params i965_cursor_wm_info = {
383 .fifo_size = I965_CURSOR_FIFO,
384 .max_wm = I965_CURSOR_MAX_WM,
385 .default_wm = I965_CURSOR_DFT_WM,
386 .guard_size = 2,
387 .cacheline_size = I915_FIFO_LINE_SIZE,
388};
389
390static const struct intel_watermark_params i945_wm_info = {
391 .fifo_size = I945_FIFO_SIZE,
392 .max_wm = I915_MAX_WM,
393 .default_wm = 1,
394 .guard_size = 2,
395 .cacheline_size = I915_FIFO_LINE_SIZE,
396};
397
398static const struct intel_watermark_params i915_wm_info = {
399 .fifo_size = I915_FIFO_SIZE,
400 .max_wm = I915_MAX_WM,
401 .default_wm = 1,
402 .guard_size = 2,
403 .cacheline_size = I915_FIFO_LINE_SIZE,
404};
405
406static const struct intel_watermark_params i830_a_wm_info = {
407 .fifo_size = I855GM_FIFO_SIZE,
408 .max_wm = I915_MAX_WM,
409 .default_wm = 1,
410 .guard_size = 2,
411 .cacheline_size = I830_FIFO_LINE_SIZE,
412};
413
414static const struct intel_watermark_params i830_bc_wm_info = {
415 .fifo_size = I855GM_FIFO_SIZE,
416 .max_wm = I915_MAX_WM / 2,
417 .default_wm = 1,
418 .guard_size = 2,
419 .cacheline_size = I830_FIFO_LINE_SIZE,
420};
421
422static const struct intel_watermark_params i845_wm_info = {
423 .fifo_size = I830_FIFO_SIZE,
424 .max_wm = I915_MAX_WM,
425 .default_wm = 1,
426 .guard_size = 2,
427 .cacheline_size = I830_FIFO_LINE_SIZE,
428};
429
430/**
431 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
432 * @pixel_rate: Pipe pixel rate in kHz
433 * @cpp: Plane bytes per pixel
434 * @latency: Memory wakeup latency in 0.1us units
435 *
436 * Compute the watermark using the method 1 or "small buffer"
437 * formula. The caller may additonally add extra cachelines
438 * to account for TLB misses and clock crossings.
439 *
440 * This method is concerned with the short term drain rate
441 * of the FIFO, ie. it does not account for blanking periods
442 * which would effectively reduce the average drain rate across
443 * a longer period. The name "small" refers to the fact the
444 * FIFO is relatively small compared to the amount of data
445 * fetched.
446 *
447 * The FIFO level vs. time graph might look something like:
448 *
449 * |\ |\
450 * | \ | \
451 * __---__---__ (- plane active, _ blanking)
452 * -> time
453 *
454 * or perhaps like this:
455 *
456 * |\|\ |\|\
457 * __----__----__ (- plane active, _ blanking)
458 * -> time
459 *
460 * Returns:
461 * The watermark in bytes
462 */
463static unsigned int intel_wm_method1(unsigned int pixel_rate,
464 unsigned int cpp,
465 unsigned int latency)
466{
467 u64 ret;
468
469 ret = mul_u32_u32(pixel_rate, cpp * latency);
470 ret = DIV_ROUND_UP_ULL(ret, 10000);
471
472 return ret;
473}
474
475/**
476 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
477 * @pixel_rate: Pipe pixel rate in kHz
478 * @htotal: Pipe horizontal total
479 * @width: Plane width in pixels
480 * @cpp: Plane bytes per pixel
481 * @latency: Memory wakeup latency in 0.1us units
482 *
483 * Compute the watermark using the method 2 or "large buffer"
484 * formula. The caller may additonally add extra cachelines
485 * to account for TLB misses and clock crossings.
486 *
487 * This method is concerned with the long term drain rate
488 * of the FIFO, ie. it does account for blanking periods
489 * which effectively reduce the average drain rate across
490 * a longer period. The name "large" refers to the fact the
491 * FIFO is relatively large compared to the amount of data
492 * fetched.
493 *
494 * The FIFO level vs. time graph might look something like:
495 *
496 * |\___ |\___
497 * | \___ | \___
498 * | \ | \
499 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
500 * -> time
501 *
502 * Returns:
503 * The watermark in bytes
504 */
505static unsigned int intel_wm_method2(unsigned int pixel_rate,
506 unsigned int htotal,
507 unsigned int width,
508 unsigned int cpp,
509 unsigned int latency)
510{
511 unsigned int ret;
512
513 /*
514 * FIXME remove once all users are computing
515 * watermarks in the correct place.
516 */
517 if (WARN_ON_ONCE(htotal == 0))
518 htotal = 1;
519
520 ret = (latency * pixel_rate) / (htotal * 10000);
521 ret = (ret + 1) * width * cpp;
522
523 return ret;
524}
525
526/**
527 * intel_calculate_wm - calculate watermark level
528 * @pixel_rate: pixel clock
529 * @wm: chip FIFO params
530 * @fifo_size: size of the FIFO buffer
531 * @cpp: bytes per pixel
532 * @latency_ns: memory latency for the platform
533 *
534 * Calculate the watermark level (the level at which the display plane will
535 * start fetching from memory again). Each chip has a different display
536 * FIFO size and allocation, so the caller needs to figure that out and pass
537 * in the correct intel_watermark_params structure.
538 *
539 * As the pixel clock runs, the FIFO will be drained at a rate that depends
540 * on the pixel size. When it reaches the watermark level, it'll start
541 * fetching FIFO line sized based chunks from memory until the FIFO fills
542 * past the watermark point. If the FIFO drains completely, a FIFO underrun
543 * will occur, and a display engine hang could result.
544 */
545static unsigned int intel_calculate_wm(int pixel_rate,
546 const struct intel_watermark_params *wm,
547 int fifo_size, int cpp,
548 unsigned int latency_ns)
549{
550 int entries, wm_size;
551
552 /*
553 * Note: we need to make sure we don't overflow for various clock &
554 * latency values.
555 * clocks go from a few thousand to several hundred thousand.
556 * latency is usually a few thousand
557 */
558 entries = intel_wm_method1(pixel_rate, cpp,
559 latency_ns / 100);
560 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
561 wm->guard_size;
562 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
563
564 wm_size = fifo_size - entries;
565 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
566
567 /* Don't promote wm_size to unsigned... */
568 if (wm_size > wm->max_wm)
569 wm_size = wm->max_wm;
570 if (wm_size <= 0)
571 wm_size = wm->default_wm;
572
573 /*
574 * Bspec seems to indicate that the value shouldn't be lower than
575 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
576 * Lets go for 8 which is the burst size since certain platforms
577 * already use a hardcoded 8 (which is what the spec says should be
578 * done).
579 */
580 if (wm_size <= 8)
581 wm_size = 8;
582
583 return wm_size;
584}
585
586static bool is_disabling(int old, int new, int threshold)
587{
588 return old >= threshold && new < threshold;
589}
590
591static bool is_enabling(int old, int new, int threshold)
592{
593 return old < threshold && new >= threshold;
594}
595
596static bool intel_crtc_active(struct intel_crtc *crtc)
597{
598 /* Be paranoid as we can arrive here with only partial
599 * state retrieved from the hardware during setup.
600 *
601 * We can ditch the adjusted_mode.crtc_clock check as soon
602 * as Haswell has gained clock readout/fastboot support.
603 *
604 * We can ditch the crtc->primary->state->fb check as soon as we can
605 * properly reconstruct framebuffers.
606 *
607 * FIXME: The intel_crtc->active here should be switched to
608 * crtc->state->active once we have proper CRTC states wired up
609 * for atomic.
610 */
611 return crtc->active && crtc->base.primary->state->fb &&
612 crtc->config->hw.adjusted_mode.crtc_clock;
613}
614
615static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
616{
617 struct intel_crtc *crtc, *enabled = NULL;
618
619 for_each_intel_crtc(&dev_priv->drm, crtc) {
620 if (intel_crtc_active(crtc)) {
621 if (enabled)
622 return NULL;
623 enabled = crtc;
624 }
625 }
626
627 return enabled;
628}
629
630static void pnv_update_wm(struct drm_i915_private *dev_priv)
631{
632 struct intel_crtc *crtc;
633 const struct cxsr_latency *latency;
634 u32 reg;
635 unsigned int wm;
636
637 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
638 dev_priv->is_ddr3,
639 dev_priv->fsb_freq,
640 dev_priv->mem_freq);
641 if (!latency) {
642 drm_dbg_kms(&dev_priv->drm,
643 "Unknown FSB/MEM found, disable CxSR\n");
644 intel_set_memory_cxsr(dev_priv, false);
645 return;
646 }
647
648 crtc = single_enabled_crtc(dev_priv);
649 if (crtc) {
650 const struct drm_framebuffer *fb =
651 crtc->base.primary->state->fb;
652 int pixel_rate = crtc->config->pixel_rate;
653 int cpp = fb->format->cpp[0];
654
655 /* Display SR */
656 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
657 pnv_display_wm.fifo_size,
658 cpp, latency->display_sr);
659 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
660 reg &= ~DSPFW_SR_MASK;
661 reg |= FW_WM(wm, SR);
662 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
663 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
664
665 /* cursor SR */
666 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
667 pnv_display_wm.fifo_size,
668 4, latency->cursor_sr);
669 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
670 FW_WM(wm, CURSOR_SR));
671
672 /* Display HPLL off SR */
673 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
674 pnv_display_hplloff_wm.fifo_size,
675 cpp, latency->display_hpll_disable);
676 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
677
678 /* cursor HPLL off SR */
679 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
680 pnv_display_hplloff_wm.fifo_size,
681 4, latency->cursor_hpll_disable);
682 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
683 reg &= ~DSPFW_HPLL_CURSOR_MASK;
684 reg |= FW_WM(wm, HPLL_CURSOR);
685 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
686 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
687
688 intel_set_memory_cxsr(dev_priv, true);
689 } else {
690 intel_set_memory_cxsr(dev_priv, false);
691 }
692}
693
694/*
695 * Documentation says:
696 * "If the line size is small, the TLB fetches can get in the way of the
697 * data fetches, causing some lag in the pixel data return which is not
698 * accounted for in the above formulas. The following adjustment only
699 * needs to be applied if eight whole lines fit in the buffer at once.
700 * The WM is adjusted upwards by the difference between the FIFO size
701 * and the size of 8 whole lines. This adjustment is always performed
702 * in the actual pixel depth regardless of whether FBC is enabled or not."
703 */
704static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
705{
706 int tlb_miss = fifo_size * 64 - width * cpp * 8;
707
708 return max(0, tlb_miss);
709}
710
711static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
712 const struct g4x_wm_values *wm)
713{
714 enum pipe pipe;
715
716 for_each_pipe(dev_priv, pipe)
717 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
718
719 intel_uncore_write(&dev_priv->uncore, DSPFW1,
720 FW_WM(wm->sr.plane, SR) |
721 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
722 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
723 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
724 intel_uncore_write(&dev_priv->uncore, DSPFW2,
725 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
726 FW_WM(wm->sr.fbc, FBC_SR) |
727 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
728 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
729 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
730 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
731 intel_uncore_write(&dev_priv->uncore, DSPFW3,
732 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
733 FW_WM(wm->sr.cursor, CURSOR_SR) |
734 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
735 FW_WM(wm->hpll.plane, HPLL_SR));
736
737 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
738}
739
740#define FW_WM_VLV(value, plane) \
741 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
742
743static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
744 const struct vlv_wm_values *wm)
745{
746 enum pipe pipe;
747
748 for_each_pipe(dev_priv, pipe) {
749 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
750
751 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
752 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
753 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
754 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
755 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
756 }
757
758 /*
759 * Zero the (unused) WM1 watermarks, and also clear all the
760 * high order bits so that there are no out of bounds values
761 * present in the registers during the reprogramming.
762 */
763 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
764 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
765 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
766 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
767 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
768
769 intel_uncore_write(&dev_priv->uncore, DSPFW1,
770 FW_WM(wm->sr.plane, SR) |
771 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
772 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
773 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
774 intel_uncore_write(&dev_priv->uncore, DSPFW2,
775 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
776 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
777 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
778 intel_uncore_write(&dev_priv->uncore, DSPFW3,
779 FW_WM(wm->sr.cursor, CURSOR_SR));
780
781 if (IS_CHERRYVIEW(dev_priv)) {
782 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
783 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
784 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
785 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
786 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
787 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
788 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
789 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
790 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
791 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
792 FW_WM(wm->sr.plane >> 9, SR_HI) |
793 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
794 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
795 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
796 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
797 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
798 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
799 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
800 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
801 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
802 } else {
803 intel_uncore_write(&dev_priv->uncore, DSPFW7,
804 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
805 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
806 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
807 FW_WM(wm->sr.plane >> 9, SR_HI) |
808 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
809 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
810 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
811 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
812 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
813 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
814 }
815
816 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
817}
818
819#undef FW_WM_VLV
820
821static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
822{
823 /* all latencies in usec */
824 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
825 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
826 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
827
828 dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
829}
830
831static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
832{
833 /*
834 * DSPCNTR[13] supposedly controls whether the
835 * primary plane can use the FIFO space otherwise
836 * reserved for the sprite plane. It's not 100% clear
837 * what the actual FIFO size is, but it looks like we
838 * can happily set both primary and sprite watermarks
839 * up to 127 cachelines. So that would seem to mean
840 * that either DSPCNTR[13] doesn't do anything, or that
841 * the total FIFO is >= 256 cachelines in size. Either
842 * way, we don't seem to have to worry about this
843 * repartitioning as the maximum watermark value the
844 * register can hold for each plane is lower than the
845 * minimum FIFO size.
846 */
847 switch (plane_id) {
848 case PLANE_CURSOR:
849 return 63;
850 case PLANE_PRIMARY:
851 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
852 case PLANE_SPRITE0:
853 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
854 default:
855 MISSING_CASE(plane_id);
856 return 0;
857 }
858}
859
860static int g4x_fbc_fifo_size(int level)
861{
862 switch (level) {
863 case G4X_WM_LEVEL_SR:
864 return 7;
865 case G4X_WM_LEVEL_HPLL:
866 return 15;
867 default:
868 MISSING_CASE(level);
869 return 0;
870 }
871}
872
873static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
874 const struct intel_plane_state *plane_state,
875 int level)
876{
877 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
878 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
879 const struct drm_display_mode *pipe_mode =
880 &crtc_state->hw.pipe_mode;
881 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
882 unsigned int pixel_rate, htotal, cpp, width, wm;
883
884 if (latency == 0)
885 return USHRT_MAX;
886
887 if (!intel_wm_plane_visible(crtc_state, plane_state))
888 return 0;
889
890 cpp = plane_state->hw.fb->format->cpp[0];
891
892 /*
893 * WaUse32BppForSRWM:ctg,elk
894 *
895 * The spec fails to list this restriction for the
896 * HPLL watermark, which seems a little strange.
897 * Let's use 32bpp for the HPLL watermark as well.
898 */
899 if (plane->id == PLANE_PRIMARY &&
900 level != G4X_WM_LEVEL_NORMAL)
901 cpp = max(cpp, 4u);
902
903 pixel_rate = crtc_state->pixel_rate;
904 htotal = pipe_mode->crtc_htotal;
905 width = drm_rect_width(&plane_state->uapi.src) >> 16;
906
907 if (plane->id == PLANE_CURSOR) {
908 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
909 } else if (plane->id == PLANE_PRIMARY &&
910 level == G4X_WM_LEVEL_NORMAL) {
911 wm = intel_wm_method1(pixel_rate, cpp, latency);
912 } else {
913 unsigned int small, large;
914
915 small = intel_wm_method1(pixel_rate, cpp, latency);
916 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
917
918 wm = min(small, large);
919 }
920
921 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
922 width, cpp);
923
924 wm = DIV_ROUND_UP(wm, 64) + 2;
925
926 return min_t(unsigned int, wm, USHRT_MAX);
927}
928
929static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
930 int level, enum plane_id plane_id, u16 value)
931{
932 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
933 bool dirty = false;
934
935 for (; level < dev_priv->display.wm.num_levels; level++) {
936 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
937
938 dirty |= raw->plane[plane_id] != value;
939 raw->plane[plane_id] = value;
940 }
941
942 return dirty;
943}
944
945static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
946 int level, u16 value)
947{
948 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
949 bool dirty = false;
950
951 /* NORMAL level doesn't have an FBC watermark */
952 level = max(level, G4X_WM_LEVEL_SR);
953
954 for (; level < dev_priv->display.wm.num_levels; level++) {
955 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
956
957 dirty |= raw->fbc != value;
958 raw->fbc = value;
959 }
960
961 return dirty;
962}
963
964static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
965 const struct intel_plane_state *plane_state,
966 u32 pri_val);
967
968static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
969 const struct intel_plane_state *plane_state)
970{
971 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
972 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
973 enum plane_id plane_id = plane->id;
974 bool dirty = false;
975 int level;
976
977 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
978 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
979 if (plane_id == PLANE_PRIMARY)
980 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
981 goto out;
982 }
983
984 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
985 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
986 int wm, max_wm;
987
988 wm = g4x_compute_wm(crtc_state, plane_state, level);
989 max_wm = g4x_plane_fifo_size(plane_id, level);
990
991 if (wm > max_wm)
992 break;
993
994 dirty |= raw->plane[plane_id] != wm;
995 raw->plane[plane_id] = wm;
996
997 if (plane_id != PLANE_PRIMARY ||
998 level == G4X_WM_LEVEL_NORMAL)
999 continue;
1000
1001 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1002 raw->plane[plane_id]);
1003 max_wm = g4x_fbc_fifo_size(level);
1004
1005 /*
1006 * FBC wm is not mandatory as we
1007 * can always just disable its use.
1008 */
1009 if (wm > max_wm)
1010 wm = USHRT_MAX;
1011
1012 dirty |= raw->fbc != wm;
1013 raw->fbc = wm;
1014 }
1015
1016 /* mark watermarks as invalid */
1017 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1018
1019 if (plane_id == PLANE_PRIMARY)
1020 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1021
1022 out:
1023 if (dirty) {
1024 drm_dbg_kms(&dev_priv->drm,
1025 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1026 plane->base.name,
1027 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1028 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1029 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1030
1031 if (plane_id == PLANE_PRIMARY)
1032 drm_dbg_kms(&dev_priv->drm,
1033 "FBC watermarks: SR=%d, HPLL=%d\n",
1034 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1035 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1036 }
1037
1038 return dirty;
1039}
1040
1041static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1042 enum plane_id plane_id, int level)
1043{
1044 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1045
1046 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1047}
1048
1049static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1050 int level)
1051{
1052 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1053
1054 if (level >= dev_priv->display.wm.num_levels)
1055 return false;
1056
1057 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1058 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1059 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1060}
1061
1062/* mark all levels starting from 'level' as invalid */
1063static void g4x_invalidate_wms(struct intel_crtc *crtc,
1064 struct g4x_wm_state *wm_state, int level)
1065{
1066 if (level <= G4X_WM_LEVEL_NORMAL) {
1067 enum plane_id plane_id;
1068
1069 for_each_plane_id_on_crtc(crtc, plane_id)
1070 wm_state->wm.plane[plane_id] = USHRT_MAX;
1071 }
1072
1073 if (level <= G4X_WM_LEVEL_SR) {
1074 wm_state->cxsr = false;
1075 wm_state->sr.cursor = USHRT_MAX;
1076 wm_state->sr.plane = USHRT_MAX;
1077 wm_state->sr.fbc = USHRT_MAX;
1078 }
1079
1080 if (level <= G4X_WM_LEVEL_HPLL) {
1081 wm_state->hpll_en = false;
1082 wm_state->hpll.cursor = USHRT_MAX;
1083 wm_state->hpll.plane = USHRT_MAX;
1084 wm_state->hpll.fbc = USHRT_MAX;
1085 }
1086}
1087
1088static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1089 int level)
1090{
1091 if (level < G4X_WM_LEVEL_SR)
1092 return false;
1093
1094 if (level >= G4X_WM_LEVEL_SR &&
1095 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1096 return false;
1097
1098 if (level >= G4X_WM_LEVEL_HPLL &&
1099 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1100 return false;
1101
1102 return true;
1103}
1104
1105static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1106{
1107 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1108 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1109 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1110 const struct g4x_pipe_wm *raw;
1111 enum plane_id plane_id;
1112 int level;
1113
1114 level = G4X_WM_LEVEL_NORMAL;
1115 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1116 goto out;
1117
1118 raw = &crtc_state->wm.g4x.raw[level];
1119 for_each_plane_id_on_crtc(crtc, plane_id)
1120 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1121
1122 level = G4X_WM_LEVEL_SR;
1123 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1124 goto out;
1125
1126 raw = &crtc_state->wm.g4x.raw[level];
1127 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1128 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1129 wm_state->sr.fbc = raw->fbc;
1130
1131 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1132
1133 level = G4X_WM_LEVEL_HPLL;
1134 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1135 goto out;
1136
1137 raw = &crtc_state->wm.g4x.raw[level];
1138 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1139 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1140 wm_state->hpll.fbc = raw->fbc;
1141
1142 wm_state->hpll_en = wm_state->cxsr;
1143
1144 level++;
1145
1146 out:
1147 if (level == G4X_WM_LEVEL_NORMAL)
1148 return -EINVAL;
1149
1150 /* invalidate the higher levels */
1151 g4x_invalidate_wms(crtc, wm_state, level);
1152
1153 /*
1154 * Determine if the FBC watermark(s) can be used. IF
1155 * this isn't the case we prefer to disable the FBC
1156 * watermark(s) rather than disable the SR/HPLL
1157 * level(s) entirely. 'level-1' is the highest valid
1158 * level here.
1159 */
1160 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1161
1162 return 0;
1163}
1164
1165static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1166 struct intel_crtc *crtc)
1167{
1168 struct intel_crtc_state *crtc_state =
1169 intel_atomic_get_new_crtc_state(state, crtc);
1170 const struct intel_plane_state *old_plane_state;
1171 const struct intel_plane_state *new_plane_state;
1172 struct intel_plane *plane;
1173 unsigned int dirty = 0;
1174 int i;
1175
1176 for_each_oldnew_intel_plane_in_state(state, plane,
1177 old_plane_state,
1178 new_plane_state, i) {
1179 if (new_plane_state->hw.crtc != &crtc->base &&
1180 old_plane_state->hw.crtc != &crtc->base)
1181 continue;
1182
1183 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1184 dirty |= BIT(plane->id);
1185 }
1186
1187 if (!dirty)
1188 return 0;
1189
1190 return _g4x_compute_pipe_wm(crtc_state);
1191}
1192
1193static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1194 struct intel_crtc *crtc)
1195{
1196 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1197 struct intel_crtc_state *new_crtc_state =
1198 intel_atomic_get_new_crtc_state(state, crtc);
1199 const struct intel_crtc_state *old_crtc_state =
1200 intel_atomic_get_old_crtc_state(state, crtc);
1201 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1202 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1203 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1204 enum plane_id plane_id;
1205
1206 if (!new_crtc_state->hw.active ||
1207 intel_crtc_needs_modeset(new_crtc_state)) {
1208 *intermediate = *optimal;
1209
1210 intermediate->cxsr = false;
1211 intermediate->hpll_en = false;
1212 goto out;
1213 }
1214
1215 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1216 !new_crtc_state->disable_cxsr;
1217 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1218 !new_crtc_state->disable_cxsr;
1219 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1220
1221 for_each_plane_id_on_crtc(crtc, plane_id) {
1222 intermediate->wm.plane[plane_id] =
1223 max(optimal->wm.plane[plane_id],
1224 active->wm.plane[plane_id]);
1225
1226 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1227 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1228 }
1229
1230 intermediate->sr.plane = max(optimal->sr.plane,
1231 active->sr.plane);
1232 intermediate->sr.cursor = max(optimal->sr.cursor,
1233 active->sr.cursor);
1234 intermediate->sr.fbc = max(optimal->sr.fbc,
1235 active->sr.fbc);
1236
1237 intermediate->hpll.plane = max(optimal->hpll.plane,
1238 active->hpll.plane);
1239 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1240 active->hpll.cursor);
1241 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1242 active->hpll.fbc);
1243
1244 drm_WARN_ON(&dev_priv->drm,
1245 (intermediate->sr.plane >
1246 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1247 intermediate->sr.cursor >
1248 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1249 intermediate->cxsr);
1250 drm_WARN_ON(&dev_priv->drm,
1251 (intermediate->sr.plane >
1252 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1253 intermediate->sr.cursor >
1254 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1255 intermediate->hpll_en);
1256
1257 drm_WARN_ON(&dev_priv->drm,
1258 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1259 intermediate->fbc_en && intermediate->cxsr);
1260 drm_WARN_ON(&dev_priv->drm,
1261 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1262 intermediate->fbc_en && intermediate->hpll_en);
1263
1264out:
1265 /*
1266 * If our intermediate WM are identical to the final WM, then we can
1267 * omit the post-vblank programming; only update if it's different.
1268 */
1269 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1270 new_crtc_state->wm.need_postvbl_update = true;
1271
1272 return 0;
1273}
1274
1275static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1276 struct g4x_wm_values *wm)
1277{
1278 struct intel_crtc *crtc;
1279 int num_active_pipes = 0;
1280
1281 wm->cxsr = true;
1282 wm->hpll_en = true;
1283 wm->fbc_en = true;
1284
1285 for_each_intel_crtc(&dev_priv->drm, crtc) {
1286 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1287
1288 if (!crtc->active)
1289 continue;
1290
1291 if (!wm_state->cxsr)
1292 wm->cxsr = false;
1293 if (!wm_state->hpll_en)
1294 wm->hpll_en = false;
1295 if (!wm_state->fbc_en)
1296 wm->fbc_en = false;
1297
1298 num_active_pipes++;
1299 }
1300
1301 if (num_active_pipes != 1) {
1302 wm->cxsr = false;
1303 wm->hpll_en = false;
1304 wm->fbc_en = false;
1305 }
1306
1307 for_each_intel_crtc(&dev_priv->drm, crtc) {
1308 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1309 enum pipe pipe = crtc->pipe;
1310
1311 wm->pipe[pipe] = wm_state->wm;
1312 if (crtc->active && wm->cxsr)
1313 wm->sr = wm_state->sr;
1314 if (crtc->active && wm->hpll_en)
1315 wm->hpll = wm_state->hpll;
1316 }
1317}
1318
1319static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1320{
1321 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1322 struct g4x_wm_values new_wm = {};
1323
1324 g4x_merge_wm(dev_priv, &new_wm);
1325
1326 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1327 return;
1328
1329 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1330 _intel_set_memory_cxsr(dev_priv, false);
1331
1332 g4x_write_wm_values(dev_priv, &new_wm);
1333
1334 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1335 _intel_set_memory_cxsr(dev_priv, true);
1336
1337 *old_wm = new_wm;
1338}
1339
1340static void g4x_initial_watermarks(struct intel_atomic_state *state,
1341 struct intel_crtc *crtc)
1342{
1343 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1344 const struct intel_crtc_state *crtc_state =
1345 intel_atomic_get_new_crtc_state(state, crtc);
1346
1347 mutex_lock(&dev_priv->display.wm.wm_mutex);
1348 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1349 g4x_program_watermarks(dev_priv);
1350 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1351}
1352
1353static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1354 struct intel_crtc *crtc)
1355{
1356 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1357 const struct intel_crtc_state *crtc_state =
1358 intel_atomic_get_new_crtc_state(state, crtc);
1359
1360 if (!crtc_state->wm.need_postvbl_update)
1361 return;
1362
1363 mutex_lock(&dev_priv->display.wm.wm_mutex);
1364 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1365 g4x_program_watermarks(dev_priv);
1366 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1367}
1368
1369/* latency must be in 0.1us units. */
1370static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1371 unsigned int htotal,
1372 unsigned int width,
1373 unsigned int cpp,
1374 unsigned int latency)
1375{
1376 unsigned int ret;
1377
1378 ret = intel_wm_method2(pixel_rate, htotal,
1379 width, cpp, latency);
1380 ret = DIV_ROUND_UP(ret, 64);
1381
1382 return ret;
1383}
1384
1385static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1386{
1387 /* all latencies in usec */
1388 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1389
1390 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1391
1392 if (IS_CHERRYVIEW(dev_priv)) {
1393 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1394 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1395
1396 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1397 }
1398}
1399
1400static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1401 const struct intel_plane_state *plane_state,
1402 int level)
1403{
1404 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1405 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1406 const struct drm_display_mode *pipe_mode =
1407 &crtc_state->hw.pipe_mode;
1408 unsigned int pixel_rate, htotal, cpp, width, wm;
1409
1410 if (dev_priv->display.wm.pri_latency[level] == 0)
1411 return USHRT_MAX;
1412
1413 if (!intel_wm_plane_visible(crtc_state, plane_state))
1414 return 0;
1415
1416 cpp = plane_state->hw.fb->format->cpp[0];
1417 pixel_rate = crtc_state->pixel_rate;
1418 htotal = pipe_mode->crtc_htotal;
1419 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1420
1421 if (plane->id == PLANE_CURSOR) {
1422 /*
1423 * FIXME the formula gives values that are
1424 * too big for the cursor FIFO, and hence we
1425 * would never be able to use cursors. For
1426 * now just hardcode the watermark.
1427 */
1428 wm = 63;
1429 } else {
1430 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1431 dev_priv->display.wm.pri_latency[level] * 10);
1432 }
1433
1434 return min_t(unsigned int, wm, USHRT_MAX);
1435}
1436
1437static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1438{
1439 return (active_planes & (BIT(PLANE_SPRITE0) |
1440 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1441}
1442
1443static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1444{
1445 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1446 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1447 const struct g4x_pipe_wm *raw =
1448 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1449 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1450 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1451 int num_active_planes = hweight8(active_planes);
1452 const int fifo_size = 511;
1453 int fifo_extra, fifo_left = fifo_size;
1454 int sprite0_fifo_extra = 0;
1455 unsigned int total_rate;
1456 enum plane_id plane_id;
1457
1458 /*
1459 * When enabling sprite0 after sprite1 has already been enabled
1460 * we tend to get an underrun unless sprite0 already has some
1461 * FIFO space allcoated. Hence we always allocate at least one
1462 * cacheline for sprite0 whenever sprite1 is enabled.
1463 *
1464 * All other plane enable sequences appear immune to this problem.
1465 */
1466 if (vlv_need_sprite0_fifo_workaround(active_planes))
1467 sprite0_fifo_extra = 1;
1468
1469 total_rate = raw->plane[PLANE_PRIMARY] +
1470 raw->plane[PLANE_SPRITE0] +
1471 raw->plane[PLANE_SPRITE1] +
1472 sprite0_fifo_extra;
1473
1474 if (total_rate > fifo_size)
1475 return -EINVAL;
1476
1477 if (total_rate == 0)
1478 total_rate = 1;
1479
1480 for_each_plane_id_on_crtc(crtc, plane_id) {
1481 unsigned int rate;
1482
1483 if ((active_planes & BIT(plane_id)) == 0) {
1484 fifo_state->plane[plane_id] = 0;
1485 continue;
1486 }
1487
1488 rate = raw->plane[plane_id];
1489 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1490 fifo_left -= fifo_state->plane[plane_id];
1491 }
1492
1493 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1494 fifo_left -= sprite0_fifo_extra;
1495
1496 fifo_state->plane[PLANE_CURSOR] = 63;
1497
1498 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1499
1500 /* spread the remainder evenly */
1501 for_each_plane_id_on_crtc(crtc, plane_id) {
1502 int plane_extra;
1503
1504 if (fifo_left == 0)
1505 break;
1506
1507 if ((active_planes & BIT(plane_id)) == 0)
1508 continue;
1509
1510 plane_extra = min(fifo_extra, fifo_left);
1511 fifo_state->plane[plane_id] += plane_extra;
1512 fifo_left -= plane_extra;
1513 }
1514
1515 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1516
1517 /* give it all to the first plane if none are active */
1518 if (active_planes == 0) {
1519 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1520 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1521 }
1522
1523 return 0;
1524}
1525
1526/* mark all levels starting from 'level' as invalid */
1527static void vlv_invalidate_wms(struct intel_crtc *crtc,
1528 struct vlv_wm_state *wm_state, int level)
1529{
1530 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1531
1532 for (; level < dev_priv->display.wm.num_levels; level++) {
1533 enum plane_id plane_id;
1534
1535 for_each_plane_id_on_crtc(crtc, plane_id)
1536 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1537
1538 wm_state->sr[level].cursor = USHRT_MAX;
1539 wm_state->sr[level].plane = USHRT_MAX;
1540 }
1541}
1542
1543static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1544{
1545 if (wm > fifo_size)
1546 return USHRT_MAX;
1547 else
1548 return fifo_size - wm;
1549}
1550
1551/*
1552 * Starting from 'level' set all higher
1553 * levels to 'value' in the "raw" watermarks.
1554 */
1555static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1556 int level, enum plane_id plane_id, u16 value)
1557{
1558 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1559 bool dirty = false;
1560
1561 for (; level < dev_priv->display.wm.num_levels; level++) {
1562 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1563
1564 dirty |= raw->plane[plane_id] != value;
1565 raw->plane[plane_id] = value;
1566 }
1567
1568 return dirty;
1569}
1570
1571static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1572 const struct intel_plane_state *plane_state)
1573{
1574 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1575 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1576 enum plane_id plane_id = plane->id;
1577 int level;
1578 bool dirty = false;
1579
1580 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1581 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1582 goto out;
1583 }
1584
1585 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1586 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1587 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1588 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1589
1590 if (wm > max_wm)
1591 break;
1592
1593 dirty |= raw->plane[plane_id] != wm;
1594 raw->plane[plane_id] = wm;
1595 }
1596
1597 /* mark all higher levels as invalid */
1598 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1599
1600out:
1601 if (dirty)
1602 drm_dbg_kms(&dev_priv->drm,
1603 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1604 plane->base.name,
1605 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1606 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1607 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1608
1609 return dirty;
1610}
1611
1612static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1613 enum plane_id plane_id, int level)
1614{
1615 const struct g4x_pipe_wm *raw =
1616 &crtc_state->wm.vlv.raw[level];
1617 const struct vlv_fifo_state *fifo_state =
1618 &crtc_state->wm.vlv.fifo_state;
1619
1620 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1621}
1622
1623static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1624{
1625 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1626 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1627 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1628 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1629}
1630
1631static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1632{
1633 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1634 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1635 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1636 const struct vlv_fifo_state *fifo_state =
1637 &crtc_state->wm.vlv.fifo_state;
1638 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1639 int num_active_planes = hweight8(active_planes);
1640 enum plane_id plane_id;
1641 int level;
1642
1643 /* initially allow all levels */
1644 wm_state->num_levels = dev_priv->display.wm.num_levels;
1645 /*
1646 * Note that enabling cxsr with no primary/sprite planes
1647 * enabled can wedge the pipe. Hence we only allow cxsr
1648 * with exactly one enabled primary/sprite plane.
1649 */
1650 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1651
1652 for (level = 0; level < wm_state->num_levels; level++) {
1653 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1654 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1655
1656 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1657 break;
1658
1659 for_each_plane_id_on_crtc(crtc, plane_id) {
1660 wm_state->wm[level].plane[plane_id] =
1661 vlv_invert_wm_value(raw->plane[plane_id],
1662 fifo_state->plane[plane_id]);
1663 }
1664
1665 wm_state->sr[level].plane =
1666 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1667 raw->plane[PLANE_SPRITE0],
1668 raw->plane[PLANE_SPRITE1]),
1669 sr_fifo_size);
1670
1671 wm_state->sr[level].cursor =
1672 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1673 63);
1674 }
1675
1676 if (level == 0)
1677 return -EINVAL;
1678
1679 /* limit to only levels we can actually handle */
1680 wm_state->num_levels = level;
1681
1682 /* invalidate the higher levels */
1683 vlv_invalidate_wms(crtc, wm_state, level);
1684
1685 return 0;
1686}
1687
1688static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1689 struct intel_crtc *crtc)
1690{
1691 struct intel_crtc_state *crtc_state =
1692 intel_atomic_get_new_crtc_state(state, crtc);
1693 const struct intel_plane_state *old_plane_state;
1694 const struct intel_plane_state *new_plane_state;
1695 struct intel_plane *plane;
1696 unsigned int dirty = 0;
1697 int i;
1698
1699 for_each_oldnew_intel_plane_in_state(state, plane,
1700 old_plane_state,
1701 new_plane_state, i) {
1702 if (new_plane_state->hw.crtc != &crtc->base &&
1703 old_plane_state->hw.crtc != &crtc->base)
1704 continue;
1705
1706 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1707 dirty |= BIT(plane->id);
1708 }
1709
1710 /*
1711 * DSPARB registers may have been reset due to the
1712 * power well being turned off. Make sure we restore
1713 * them to a consistent state even if no primary/sprite
1714 * planes are initially active. We also force a FIFO
1715 * recomputation so that we are sure to sanitize the
1716 * FIFO setting we took over from the BIOS even if there
1717 * are no active planes on the crtc.
1718 */
1719 if (intel_crtc_needs_modeset(crtc_state))
1720 dirty = ~0;
1721
1722 if (!dirty)
1723 return 0;
1724
1725 /* cursor changes don't warrant a FIFO recompute */
1726 if (dirty & ~BIT(PLANE_CURSOR)) {
1727 const struct intel_crtc_state *old_crtc_state =
1728 intel_atomic_get_old_crtc_state(state, crtc);
1729 const struct vlv_fifo_state *old_fifo_state =
1730 &old_crtc_state->wm.vlv.fifo_state;
1731 const struct vlv_fifo_state *new_fifo_state =
1732 &crtc_state->wm.vlv.fifo_state;
1733 int ret;
1734
1735 ret = vlv_compute_fifo(crtc_state);
1736 if (ret)
1737 return ret;
1738
1739 if (intel_crtc_needs_modeset(crtc_state) ||
1740 memcmp(old_fifo_state, new_fifo_state,
1741 sizeof(*new_fifo_state)) != 0)
1742 crtc_state->fifo_changed = true;
1743 }
1744
1745 return _vlv_compute_pipe_wm(crtc_state);
1746}
1747
1748#define VLV_FIFO(plane, value) \
1749 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1750
1751static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1752 struct intel_crtc *crtc)
1753{
1754 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1755 struct intel_uncore *uncore = &dev_priv->uncore;
1756 const struct intel_crtc_state *crtc_state =
1757 intel_atomic_get_new_crtc_state(state, crtc);
1758 const struct vlv_fifo_state *fifo_state =
1759 &crtc_state->wm.vlv.fifo_state;
1760 int sprite0_start, sprite1_start, fifo_size;
1761 u32 dsparb, dsparb2, dsparb3;
1762
1763 if (!crtc_state->fifo_changed)
1764 return;
1765
1766 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1767 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1768 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1769
1770 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1771 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1772
1773 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1774
1775 /*
1776 * uncore.lock serves a double purpose here. It allows us to
1777 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1778 * it protects the DSPARB registers from getting clobbered by
1779 * parallel updates from multiple pipes.
1780 *
1781 * intel_pipe_update_start() has already disabled interrupts
1782 * for us, so a plain spin_lock() is sufficient here.
1783 */
1784 spin_lock(&uncore->lock);
1785
1786 switch (crtc->pipe) {
1787 case PIPE_A:
1788 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1789 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1790
1791 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1792 VLV_FIFO(SPRITEB, 0xff));
1793 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1794 VLV_FIFO(SPRITEB, sprite1_start));
1795
1796 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1797 VLV_FIFO(SPRITEB_HI, 0x1));
1798 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1799 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1800
1801 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1802 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1803 break;
1804 case PIPE_B:
1805 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1806 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1807
1808 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1809 VLV_FIFO(SPRITED, 0xff));
1810 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1811 VLV_FIFO(SPRITED, sprite1_start));
1812
1813 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1814 VLV_FIFO(SPRITED_HI, 0xff));
1815 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1816 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1817
1818 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1819 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1820 break;
1821 case PIPE_C:
1822 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1823 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1824
1825 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1826 VLV_FIFO(SPRITEF, 0xff));
1827 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1828 VLV_FIFO(SPRITEF, sprite1_start));
1829
1830 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1831 VLV_FIFO(SPRITEF_HI, 0xff));
1832 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1833 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1834
1835 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1836 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1837 break;
1838 default:
1839 break;
1840 }
1841
1842 intel_uncore_posting_read_fw(uncore, DSPARB);
1843
1844 spin_unlock(&uncore->lock);
1845}
1846
1847#undef VLV_FIFO
1848
1849static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1850 struct intel_crtc *crtc)
1851{
1852 struct intel_crtc_state *new_crtc_state =
1853 intel_atomic_get_new_crtc_state(state, crtc);
1854 const struct intel_crtc_state *old_crtc_state =
1855 intel_atomic_get_old_crtc_state(state, crtc);
1856 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1857 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1858 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1859 int level;
1860
1861 if (!new_crtc_state->hw.active ||
1862 intel_crtc_needs_modeset(new_crtc_state)) {
1863 *intermediate = *optimal;
1864
1865 intermediate->cxsr = false;
1866 goto out;
1867 }
1868
1869 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1870 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1871 !new_crtc_state->disable_cxsr;
1872
1873 for (level = 0; level < intermediate->num_levels; level++) {
1874 enum plane_id plane_id;
1875
1876 for_each_plane_id_on_crtc(crtc, plane_id) {
1877 intermediate->wm[level].plane[plane_id] =
1878 min(optimal->wm[level].plane[plane_id],
1879 active->wm[level].plane[plane_id]);
1880 }
1881
1882 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1883 active->sr[level].plane);
1884 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1885 active->sr[level].cursor);
1886 }
1887
1888 vlv_invalidate_wms(crtc, intermediate, level);
1889
1890out:
1891 /*
1892 * If our intermediate WM are identical to the final WM, then we can
1893 * omit the post-vblank programming; only update if it's different.
1894 */
1895 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1896 new_crtc_state->wm.need_postvbl_update = true;
1897
1898 return 0;
1899}
1900
1901static void vlv_merge_wm(struct drm_i915_private *dev_priv,
1902 struct vlv_wm_values *wm)
1903{
1904 struct intel_crtc *crtc;
1905 int num_active_pipes = 0;
1906
1907 wm->level = dev_priv->display.wm.num_levels - 1;
1908 wm->cxsr = true;
1909
1910 for_each_intel_crtc(&dev_priv->drm, crtc) {
1911 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1912
1913 if (!crtc->active)
1914 continue;
1915
1916 if (!wm_state->cxsr)
1917 wm->cxsr = false;
1918
1919 num_active_pipes++;
1920 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1921 }
1922
1923 if (num_active_pipes != 1)
1924 wm->cxsr = false;
1925
1926 if (num_active_pipes > 1)
1927 wm->level = VLV_WM_LEVEL_PM2;
1928
1929 for_each_intel_crtc(&dev_priv->drm, crtc) {
1930 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1931 enum pipe pipe = crtc->pipe;
1932
1933 wm->pipe[pipe] = wm_state->wm[wm->level];
1934 if (crtc->active && wm->cxsr)
1935 wm->sr = wm_state->sr[wm->level];
1936
1937 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
1938 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
1939 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
1940 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
1941 }
1942}
1943
1944static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
1945{
1946 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
1947 struct vlv_wm_values new_wm = {};
1948
1949 vlv_merge_wm(dev_priv, &new_wm);
1950
1951 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1952 return;
1953
1954 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1955 chv_set_memory_dvfs(dev_priv, false);
1956
1957 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1958 chv_set_memory_pm5(dev_priv, false);
1959
1960 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1961 _intel_set_memory_cxsr(dev_priv, false);
1962
1963 vlv_write_wm_values(dev_priv, &new_wm);
1964
1965 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1966 _intel_set_memory_cxsr(dev_priv, true);
1967
1968 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1969 chv_set_memory_pm5(dev_priv, true);
1970
1971 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1972 chv_set_memory_dvfs(dev_priv, true);
1973
1974 *old_wm = new_wm;
1975}
1976
1977static void vlv_initial_watermarks(struct intel_atomic_state *state,
1978 struct intel_crtc *crtc)
1979{
1980 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1981 const struct intel_crtc_state *crtc_state =
1982 intel_atomic_get_new_crtc_state(state, crtc);
1983
1984 mutex_lock(&dev_priv->display.wm.wm_mutex);
1985 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
1986 vlv_program_watermarks(dev_priv);
1987 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1988}
1989
1990static void vlv_optimize_watermarks(struct intel_atomic_state *state,
1991 struct intel_crtc *crtc)
1992{
1993 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1994 const struct intel_crtc_state *crtc_state =
1995 intel_atomic_get_new_crtc_state(state, crtc);
1996
1997 if (!crtc_state->wm.need_postvbl_update)
1998 return;
1999
2000 mutex_lock(&dev_priv->display.wm.wm_mutex);
2001 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2002 vlv_program_watermarks(dev_priv);
2003 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2004}
2005
2006static void i965_update_wm(struct drm_i915_private *dev_priv)
2007{
2008 struct intel_crtc *crtc;
2009 int srwm = 1;
2010 int cursor_sr = 16;
2011 bool cxsr_enabled;
2012
2013 /* Calc sr entries for one plane configs */
2014 crtc = single_enabled_crtc(dev_priv);
2015 if (crtc) {
2016 /* self-refresh has much higher latency */
2017 static const int sr_latency_ns = 12000;
2018 const struct drm_display_mode *pipe_mode =
2019 &crtc->config->hw.pipe_mode;
2020 const struct drm_framebuffer *fb =
2021 crtc->base.primary->state->fb;
2022 int pixel_rate = crtc->config->pixel_rate;
2023 int htotal = pipe_mode->crtc_htotal;
2024 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2025 int cpp = fb->format->cpp[0];
2026 int entries;
2027
2028 entries = intel_wm_method2(pixel_rate, htotal,
2029 width, cpp, sr_latency_ns / 100);
2030 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2031 srwm = I965_FIFO_SIZE - entries;
2032 if (srwm < 0)
2033 srwm = 1;
2034 srwm &= 0x1ff;
2035 drm_dbg_kms(&dev_priv->drm,
2036 "self-refresh entries: %d, wm: %d\n",
2037 entries, srwm);
2038
2039 entries = intel_wm_method2(pixel_rate, htotal,
2040 crtc->base.cursor->state->crtc_w, 4,
2041 sr_latency_ns / 100);
2042 entries = DIV_ROUND_UP(entries,
2043 i965_cursor_wm_info.cacheline_size) +
2044 i965_cursor_wm_info.guard_size;
2045
2046 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2047 if (cursor_sr > i965_cursor_wm_info.max_wm)
2048 cursor_sr = i965_cursor_wm_info.max_wm;
2049
2050 drm_dbg_kms(&dev_priv->drm,
2051 "self-refresh watermark: display plane %d "
2052 "cursor %d\n", srwm, cursor_sr);
2053
2054 cxsr_enabled = true;
2055 } else {
2056 cxsr_enabled = false;
2057 /* Turn off self refresh if both pipes are enabled */
2058 intel_set_memory_cxsr(dev_priv, false);
2059 }
2060
2061 drm_dbg_kms(&dev_priv->drm,
2062 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2063 srwm);
2064
2065 /* 965 has limitations... */
2066 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2067 FW_WM(8, CURSORB) |
2068 FW_WM(8, PLANEB) |
2069 FW_WM(8, PLANEA));
2070 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2071 FW_WM(8, PLANEC_OLD));
2072 /* update cursor SR watermark */
2073 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2074
2075 if (cxsr_enabled)
2076 intel_set_memory_cxsr(dev_priv, true);
2077}
2078
2079#undef FW_WM
2080
2081static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2082 enum i9xx_plane_id i9xx_plane)
2083{
2084 struct intel_plane *plane;
2085
2086 for_each_intel_plane(&i915->drm, plane) {
2087 if (plane->id == PLANE_PRIMARY &&
2088 plane->i9xx_plane == i9xx_plane)
2089 return intel_crtc_for_pipe(i915, plane->pipe);
2090 }
2091
2092 return NULL;
2093}
2094
2095static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2096{
2097 const struct intel_watermark_params *wm_info;
2098 u32 fwater_lo;
2099 u32 fwater_hi;
2100 int cwm, srwm = 1;
2101 int fifo_size;
2102 int planea_wm, planeb_wm;
2103 struct intel_crtc *crtc;
2104
2105 if (IS_I945GM(dev_priv))
2106 wm_info = &i945_wm_info;
2107 else if (DISPLAY_VER(dev_priv) != 2)
2108 wm_info = &i915_wm_info;
2109 else
2110 wm_info = &i830_a_wm_info;
2111
2112 if (DISPLAY_VER(dev_priv) == 2)
2113 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2114 else
2115 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2116 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2117 if (intel_crtc_active(crtc)) {
2118 const struct drm_framebuffer *fb =
2119 crtc->base.primary->state->fb;
2120 int cpp;
2121
2122 if (DISPLAY_VER(dev_priv) == 2)
2123 cpp = 4;
2124 else
2125 cpp = fb->format->cpp[0];
2126
2127 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2128 wm_info, fifo_size, cpp,
2129 pessimal_latency_ns);
2130 } else {
2131 planea_wm = fifo_size - wm_info->guard_size;
2132 if (planea_wm > (long)wm_info->max_wm)
2133 planea_wm = wm_info->max_wm;
2134 }
2135
2136 if (DISPLAY_VER(dev_priv) == 2)
2137 wm_info = &i830_bc_wm_info;
2138
2139 if (DISPLAY_VER(dev_priv) == 2)
2140 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2141 else
2142 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2143 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2144 if (intel_crtc_active(crtc)) {
2145 const struct drm_framebuffer *fb =
2146 crtc->base.primary->state->fb;
2147 int cpp;
2148
2149 if (DISPLAY_VER(dev_priv) == 2)
2150 cpp = 4;
2151 else
2152 cpp = fb->format->cpp[0];
2153
2154 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2155 wm_info, fifo_size, cpp,
2156 pessimal_latency_ns);
2157 } else {
2158 planeb_wm = fifo_size - wm_info->guard_size;
2159 if (planeb_wm > (long)wm_info->max_wm)
2160 planeb_wm = wm_info->max_wm;
2161 }
2162
2163 drm_dbg_kms(&dev_priv->drm,
2164 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2165
2166 crtc = single_enabled_crtc(dev_priv);
2167 if (IS_I915GM(dev_priv) && crtc) {
2168 struct drm_i915_gem_object *obj;
2169
2170 obj = intel_fb_obj(crtc->base.primary->state->fb);
2171
2172 /* self-refresh seems busted with untiled */
2173 if (!i915_gem_object_is_tiled(obj))
2174 crtc = NULL;
2175 }
2176
2177 /*
2178 * Overlay gets an aggressive default since video jitter is bad.
2179 */
2180 cwm = 2;
2181
2182 /* Play safe and disable self-refresh before adjusting watermarks. */
2183 intel_set_memory_cxsr(dev_priv, false);
2184
2185 /* Calc sr entries for one plane configs */
2186 if (HAS_FW_BLC(dev_priv) && crtc) {
2187 /* self-refresh has much higher latency */
2188 static const int sr_latency_ns = 6000;
2189 const struct drm_display_mode *pipe_mode =
2190 &crtc->config->hw.pipe_mode;
2191 const struct drm_framebuffer *fb =
2192 crtc->base.primary->state->fb;
2193 int pixel_rate = crtc->config->pixel_rate;
2194 int htotal = pipe_mode->crtc_htotal;
2195 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2196 int cpp;
2197 int entries;
2198
2199 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2200 cpp = 4;
2201 else
2202 cpp = fb->format->cpp[0];
2203
2204 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2205 sr_latency_ns / 100);
2206 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2207 drm_dbg_kms(&dev_priv->drm,
2208 "self-refresh entries: %d\n", entries);
2209 srwm = wm_info->fifo_size - entries;
2210 if (srwm < 0)
2211 srwm = 1;
2212
2213 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2214 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2215 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2216 else
2217 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2218 }
2219
2220 drm_dbg_kms(&dev_priv->drm,
2221 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2222 planea_wm, planeb_wm, cwm, srwm);
2223
2224 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2225 fwater_hi = (cwm & 0x1f);
2226
2227 /* Set request length to 8 cachelines per fetch */
2228 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2229 fwater_hi = fwater_hi | (1 << 8);
2230
2231 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2232 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2233
2234 if (crtc)
2235 intel_set_memory_cxsr(dev_priv, true);
2236}
2237
2238static void i845_update_wm(struct drm_i915_private *dev_priv)
2239{
2240 struct intel_crtc *crtc;
2241 u32 fwater_lo;
2242 int planea_wm;
2243
2244 crtc = single_enabled_crtc(dev_priv);
2245 if (crtc == NULL)
2246 return;
2247
2248 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2249 &i845_wm_info,
2250 i845_get_fifo_size(dev_priv, PLANE_A),
2251 4, pessimal_latency_ns);
2252 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2253 fwater_lo |= (3<<8) | planea_wm;
2254
2255 drm_dbg_kms(&dev_priv->drm,
2256 "Setting FIFO watermarks - A: %d\n", planea_wm);
2257
2258 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2259}
2260
2261/* latency must be in 0.1us units. */
2262static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2263 unsigned int cpp,
2264 unsigned int latency)
2265{
2266 unsigned int ret;
2267
2268 ret = intel_wm_method1(pixel_rate, cpp, latency);
2269 ret = DIV_ROUND_UP(ret, 64) + 2;
2270
2271 return ret;
2272}
2273
2274/* latency must be in 0.1us units. */
2275static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2276 unsigned int htotal,
2277 unsigned int width,
2278 unsigned int cpp,
2279 unsigned int latency)
2280{
2281 unsigned int ret;
2282
2283 ret = intel_wm_method2(pixel_rate, htotal,
2284 width, cpp, latency);
2285 ret = DIV_ROUND_UP(ret, 64) + 2;
2286
2287 return ret;
2288}
2289
2290static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2291{
2292 /*
2293 * Neither of these should be possible since this function shouldn't be
2294 * called if the CRTC is off or the plane is invisible. But let's be
2295 * extra paranoid to avoid a potential divide-by-zero if we screw up
2296 * elsewhere in the driver.
2297 */
2298 if (WARN_ON(!cpp))
2299 return 0;
2300 if (WARN_ON(!horiz_pixels))
2301 return 0;
2302
2303 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2304}
2305
2306struct ilk_wm_maximums {
2307 u16 pri;
2308 u16 spr;
2309 u16 cur;
2310 u16 fbc;
2311};
2312
2313/*
2314 * For both WM_PIPE and WM_LP.
2315 * mem_value must be in 0.1us units.
2316 */
2317static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2318 const struct intel_plane_state *plane_state,
2319 u32 mem_value, bool is_lp)
2320{
2321 u32 method1, method2;
2322 int cpp;
2323
2324 if (mem_value == 0)
2325 return U32_MAX;
2326
2327 if (!intel_wm_plane_visible(crtc_state, plane_state))
2328 return 0;
2329
2330 cpp = plane_state->hw.fb->format->cpp[0];
2331
2332 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2333
2334 if (!is_lp)
2335 return method1;
2336
2337 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2338 crtc_state->hw.pipe_mode.crtc_htotal,
2339 drm_rect_width(&plane_state->uapi.src) >> 16,
2340 cpp, mem_value);
2341
2342 return min(method1, method2);
2343}
2344
2345/*
2346 * For both WM_PIPE and WM_LP.
2347 * mem_value must be in 0.1us units.
2348 */
2349static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2350 const struct intel_plane_state *plane_state,
2351 u32 mem_value)
2352{
2353 u32 method1, method2;
2354 int cpp;
2355
2356 if (mem_value == 0)
2357 return U32_MAX;
2358
2359 if (!intel_wm_plane_visible(crtc_state, plane_state))
2360 return 0;
2361
2362 cpp = plane_state->hw.fb->format->cpp[0];
2363
2364 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2365 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2366 crtc_state->hw.pipe_mode.crtc_htotal,
2367 drm_rect_width(&plane_state->uapi.src) >> 16,
2368 cpp, mem_value);
2369 return min(method1, method2);
2370}
2371
2372/*
2373 * For both WM_PIPE and WM_LP.
2374 * mem_value must be in 0.1us units.
2375 */
2376static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2377 const struct intel_plane_state *plane_state,
2378 u32 mem_value)
2379{
2380 int cpp;
2381
2382 if (mem_value == 0)
2383 return U32_MAX;
2384
2385 if (!intel_wm_plane_visible(crtc_state, plane_state))
2386 return 0;
2387
2388 cpp = plane_state->hw.fb->format->cpp[0];
2389
2390 return ilk_wm_method2(crtc_state->pixel_rate,
2391 crtc_state->hw.pipe_mode.crtc_htotal,
2392 drm_rect_width(&plane_state->uapi.src) >> 16,
2393 cpp, mem_value);
2394}
2395
2396/* Only for WM_LP. */
2397static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2398 const struct intel_plane_state *plane_state,
2399 u32 pri_val)
2400{
2401 int cpp;
2402
2403 if (!intel_wm_plane_visible(crtc_state, plane_state))
2404 return 0;
2405
2406 cpp = plane_state->hw.fb->format->cpp[0];
2407
2408 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2409 cpp);
2410}
2411
2412static unsigned int
2413ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2414{
2415 if (DISPLAY_VER(dev_priv) >= 8)
2416 return 3072;
2417 else if (DISPLAY_VER(dev_priv) >= 7)
2418 return 768;
2419 else
2420 return 512;
2421}
2422
2423static unsigned int
2424ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2425 int level, bool is_sprite)
2426{
2427 if (DISPLAY_VER(dev_priv) >= 8)
2428 /* BDW primary/sprite plane watermarks */
2429 return level == 0 ? 255 : 2047;
2430 else if (DISPLAY_VER(dev_priv) >= 7)
2431 /* IVB/HSW primary/sprite plane watermarks */
2432 return level == 0 ? 127 : 1023;
2433 else if (!is_sprite)
2434 /* ILK/SNB primary plane watermarks */
2435 return level == 0 ? 127 : 511;
2436 else
2437 /* ILK/SNB sprite plane watermarks */
2438 return level == 0 ? 63 : 255;
2439}
2440
2441static unsigned int
2442ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2443{
2444 if (DISPLAY_VER(dev_priv) >= 7)
2445 return level == 0 ? 63 : 255;
2446 else
2447 return level == 0 ? 31 : 63;
2448}
2449
2450static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2451{
2452 if (DISPLAY_VER(dev_priv) >= 8)
2453 return 31;
2454 else
2455 return 15;
2456}
2457
2458/* Calculate the maximum primary/sprite plane watermark */
2459static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2460 int level,
2461 const struct intel_wm_config *config,
2462 enum intel_ddb_partitioning ddb_partitioning,
2463 bool is_sprite)
2464{
2465 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2466
2467 /* if sprites aren't enabled, sprites get nothing */
2468 if (is_sprite && !config->sprites_enabled)
2469 return 0;
2470
2471 /* HSW allows LP1+ watermarks even with multiple pipes */
2472 if (level == 0 || config->num_pipes_active > 1) {
2473 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2474
2475 /*
2476 * For some reason the non self refresh
2477 * FIFO size is only half of the self
2478 * refresh FIFO size on ILK/SNB.
2479 */
2480 if (DISPLAY_VER(dev_priv) < 7)
2481 fifo_size /= 2;
2482 }
2483
2484 if (config->sprites_enabled) {
2485 /* level 0 is always calculated with 1:1 split */
2486 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2487 if (is_sprite)
2488 fifo_size *= 5;
2489 fifo_size /= 6;
2490 } else {
2491 fifo_size /= 2;
2492 }
2493 }
2494
2495 /* clamp to max that the registers can hold */
2496 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2497}
2498
2499/* Calculate the maximum cursor plane watermark */
2500static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2501 int level,
2502 const struct intel_wm_config *config)
2503{
2504 /* HSW LP1+ watermarks w/ multiple pipes */
2505 if (level > 0 && config->num_pipes_active > 1)
2506 return 64;
2507
2508 /* otherwise just report max that registers can hold */
2509 return ilk_cursor_wm_reg_max(dev_priv, level);
2510}
2511
2512static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2513 int level,
2514 const struct intel_wm_config *config,
2515 enum intel_ddb_partitioning ddb_partitioning,
2516 struct ilk_wm_maximums *max)
2517{
2518 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2519 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2520 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2521 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2522}
2523
2524static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2525 int level,
2526 struct ilk_wm_maximums *max)
2527{
2528 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2529 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2530 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2531 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2532}
2533
2534static bool ilk_validate_wm_level(int level,
2535 const struct ilk_wm_maximums *max,
2536 struct intel_wm_level *result)
2537{
2538 bool ret;
2539
2540 /* already determined to be invalid? */
2541 if (!result->enable)
2542 return false;
2543
2544 result->enable = result->pri_val <= max->pri &&
2545 result->spr_val <= max->spr &&
2546 result->cur_val <= max->cur;
2547
2548 ret = result->enable;
2549
2550 /*
2551 * HACK until we can pre-compute everything,
2552 * and thus fail gracefully if LP0 watermarks
2553 * are exceeded...
2554 */
2555 if (level == 0 && !result->enable) {
2556 if (result->pri_val > max->pri)
2557 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2558 level, result->pri_val, max->pri);
2559 if (result->spr_val > max->spr)
2560 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2561 level, result->spr_val, max->spr);
2562 if (result->cur_val > max->cur)
2563 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2564 level, result->cur_val, max->cur);
2565
2566 result->pri_val = min_t(u32, result->pri_val, max->pri);
2567 result->spr_val = min_t(u32, result->spr_val, max->spr);
2568 result->cur_val = min_t(u32, result->cur_val, max->cur);
2569 result->enable = true;
2570 }
2571
2572 return ret;
2573}
2574
2575static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2576 const struct intel_crtc *crtc,
2577 int level,
2578 struct intel_crtc_state *crtc_state,
2579 const struct intel_plane_state *pristate,
2580 const struct intel_plane_state *sprstate,
2581 const struct intel_plane_state *curstate,
2582 struct intel_wm_level *result)
2583{
2584 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2585 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2586 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2587
2588 /* WM1+ latency values stored in 0.5us units */
2589 if (level > 0) {
2590 pri_latency *= 5;
2591 spr_latency *= 5;
2592 cur_latency *= 5;
2593 }
2594
2595 if (pristate) {
2596 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2597 pri_latency, level);
2598 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2599 }
2600
2601 if (sprstate)
2602 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2603
2604 if (curstate)
2605 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2606
2607 result->enable = true;
2608}
2609
2610static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2611{
2612 u64 sskpd;
2613
2614 i915->display.wm.num_levels = 5;
2615
2616 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2617
2618 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2619 if (wm[0] == 0)
2620 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2621 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2622 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2623 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2624 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2625}
2626
2627static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2628{
2629 u32 sskpd;
2630
2631 i915->display.wm.num_levels = 4;
2632
2633 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2634
2635 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2636 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2637 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2638 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2639}
2640
2641static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2642{
2643 u32 mltr;
2644
2645 i915->display.wm.num_levels = 3;
2646
2647 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2648
2649 /* ILK primary LP0 latency is 700 ns */
2650 wm[0] = 7;
2651 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2652 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2653}
2654
2655static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2656 u16 wm[5])
2657{
2658 /* ILK sprite LP0 latency is 1300 ns */
2659 if (DISPLAY_VER(dev_priv) == 5)
2660 wm[0] = 13;
2661}
2662
2663static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2664 u16 wm[5])
2665{
2666 /* ILK cursor LP0 latency is 1300 ns */
2667 if (DISPLAY_VER(dev_priv) == 5)
2668 wm[0] = 13;
2669}
2670
2671static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2672 u16 wm[5], u16 min)
2673{
2674 int level;
2675
2676 if (wm[0] >= min)
2677 return false;
2678
2679 wm[0] = max(wm[0], min);
2680 for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2681 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2682
2683 return true;
2684}
2685
2686static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2687{
2688 bool changed;
2689
2690 /*
2691 * The BIOS provided WM memory latency values are often
2692 * inadequate for high resolution displays. Adjust them.
2693 */
2694 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2695 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2696 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2697
2698 if (!changed)
2699 return;
2700
2701 drm_dbg_kms(&dev_priv->drm,
2702 "WM latency values increased to avoid potential underruns\n");
2703 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2704 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2705 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2706}
2707
2708static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2709{
2710 /*
2711 * On some SNB machines (Thinkpad X220 Tablet at least)
2712 * LP3 usage can cause vblank interrupts to be lost.
2713 * The DEIIR bit will go high but it looks like the CPU
2714 * never gets interrupted.
2715 *
2716 * It's not clear whether other interrupt source could
2717 * be affected or if this is somehow limited to vblank
2718 * interrupts only. To play it safe we disable LP3
2719 * watermarks entirely.
2720 */
2721 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2722 dev_priv->display.wm.spr_latency[3] == 0 &&
2723 dev_priv->display.wm.cur_latency[3] == 0)
2724 return;
2725
2726 dev_priv->display.wm.pri_latency[3] = 0;
2727 dev_priv->display.wm.spr_latency[3] = 0;
2728 dev_priv->display.wm.cur_latency[3] = 0;
2729
2730 drm_dbg_kms(&dev_priv->drm,
2731 "LP3 watermarks disabled due to potential for lost interrupts\n");
2732 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2733 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2734 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2735}
2736
2737static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2738{
2739 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2740 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2741 else if (DISPLAY_VER(dev_priv) >= 6)
2742 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2743 else
2744 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2745
2746 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2747 sizeof(dev_priv->display.wm.pri_latency));
2748 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2749 sizeof(dev_priv->display.wm.pri_latency));
2750
2751 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2752 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2753
2754 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2755 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2756 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2757
2758 if (DISPLAY_VER(dev_priv) == 6) {
2759 snb_wm_latency_quirk(dev_priv);
2760 snb_wm_lp3_irq_quirk(dev_priv);
2761 }
2762}
2763
2764static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
2765 struct intel_pipe_wm *pipe_wm)
2766{
2767 /* LP0 watermark maximums depend on this pipe alone */
2768 const struct intel_wm_config config = {
2769 .num_pipes_active = 1,
2770 .sprites_enabled = pipe_wm->sprites_enabled,
2771 .sprites_scaled = pipe_wm->sprites_scaled,
2772 };
2773 struct ilk_wm_maximums max;
2774
2775 /* LP0 watermarks always use 1/2 DDB partitioning */
2776 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2777
2778 /* At least LP0 must be valid */
2779 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
2780 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2781 return false;
2782 }
2783
2784 return true;
2785}
2786
2787/* Compute new watermarks for the pipe */
2788static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2789 struct intel_crtc *crtc)
2790{
2791 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2792 struct intel_crtc_state *crtc_state =
2793 intel_atomic_get_new_crtc_state(state, crtc);
2794 struct intel_pipe_wm *pipe_wm;
2795 struct intel_plane *plane;
2796 const struct intel_plane_state *plane_state;
2797 const struct intel_plane_state *pristate = NULL;
2798 const struct intel_plane_state *sprstate = NULL;
2799 const struct intel_plane_state *curstate = NULL;
2800 struct ilk_wm_maximums max;
2801 int level, usable_level;
2802
2803 pipe_wm = &crtc_state->wm.ilk.optimal;
2804
2805 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2806 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2807 pristate = plane_state;
2808 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2809 sprstate = plane_state;
2810 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2811 curstate = plane_state;
2812 }
2813
2814 pipe_wm->pipe_enabled = crtc_state->hw.active;
2815 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2816 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2817
2818 usable_level = dev_priv->display.wm.num_levels - 1;
2819
2820 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2821 if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)
2822 usable_level = 1;
2823
2824 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2825 if (pipe_wm->sprites_scaled)
2826 usable_level = 0;
2827
2828 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2829 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2830 pristate, sprstate, curstate, &pipe_wm->wm[0]);
2831
2832 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2833 return -EINVAL;
2834
2835 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2836
2837 for (level = 1; level <= usable_level; level++) {
2838 struct intel_wm_level *wm = &pipe_wm->wm[level];
2839
2840 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2841 pristate, sprstate, curstate, wm);
2842
2843 /*
2844 * Disable any watermark level that exceeds the
2845 * register maximums since such watermarks are
2846 * always invalid.
2847 */
2848 if (!ilk_validate_wm_level(level, &max, wm)) {
2849 memset(wm, 0, sizeof(*wm));
2850 break;
2851 }
2852 }
2853
2854 return 0;
2855}
2856
2857/*
2858 * Build a set of 'intermediate' watermark values that satisfy both the old
2859 * state and the new state. These can be programmed to the hardware
2860 * immediately.
2861 */
2862static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2863 struct intel_crtc *crtc)
2864{
2865 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2866 struct intel_crtc_state *new_crtc_state =
2867 intel_atomic_get_new_crtc_state(state, crtc);
2868 const struct intel_crtc_state *old_crtc_state =
2869 intel_atomic_get_old_crtc_state(state, crtc);
2870 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
2871 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
2872 int level;
2873
2874 /*
2875 * Start with the final, target watermarks, then combine with the
2876 * currently active watermarks to get values that are safe both before
2877 * and after the vblank.
2878 */
2879 *a = new_crtc_state->wm.ilk.optimal;
2880 if (!new_crtc_state->hw.active ||
2881 intel_crtc_needs_modeset(new_crtc_state) ||
2882 state->skip_intermediate_wm)
2883 return 0;
2884
2885 a->pipe_enabled |= b->pipe_enabled;
2886 a->sprites_enabled |= b->sprites_enabled;
2887 a->sprites_scaled |= b->sprites_scaled;
2888
2889 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
2890 struct intel_wm_level *a_wm = &a->wm[level];
2891 const struct intel_wm_level *b_wm = &b->wm[level];
2892
2893 a_wm->enable &= b_wm->enable;
2894 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
2895 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
2896 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
2897 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
2898 }
2899
2900 /*
2901 * We need to make sure that these merged watermark values are
2902 * actually a valid configuration themselves. If they're not,
2903 * there's no safe way to transition from the old state to
2904 * the new state, so we need to fail the atomic transaction.
2905 */
2906 if (!ilk_validate_pipe_wm(dev_priv, a))
2907 return -EINVAL;
2908
2909 /*
2910 * If our intermediate WM are identical to the final WM, then we can
2911 * omit the post-vblank programming; only update if it's different.
2912 */
2913 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
2914 new_crtc_state->wm.need_postvbl_update = true;
2915
2916 return 0;
2917}
2918
2919/*
2920 * Merge the watermarks from all active pipes for a specific level.
2921 */
2922static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
2923 int level,
2924 struct intel_wm_level *ret_wm)
2925{
2926 const struct intel_crtc *crtc;
2927
2928 ret_wm->enable = true;
2929
2930 for_each_intel_crtc(&dev_priv->drm, crtc) {
2931 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
2932 const struct intel_wm_level *wm = &active->wm[level];
2933
2934 if (!active->pipe_enabled)
2935 continue;
2936
2937 /*
2938 * The watermark values may have been used in the past,
2939 * so we must maintain them in the registers for some
2940 * time even if the level is now disabled.
2941 */
2942 if (!wm->enable)
2943 ret_wm->enable = false;
2944
2945 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2946 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2947 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2948 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2949 }
2950}
2951
2952/*
2953 * Merge all low power watermarks for all active pipes.
2954 */
2955static void ilk_wm_merge(struct drm_i915_private *dev_priv,
2956 const struct intel_wm_config *config,
2957 const struct ilk_wm_maximums *max,
2958 struct intel_pipe_wm *merged)
2959{
2960 int level, num_levels = dev_priv->display.wm.num_levels;
2961 int last_enabled_level = num_levels - 1;
2962
2963 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2964 if ((DISPLAY_VER(dev_priv) < 7 || IS_IVYBRIDGE(dev_priv)) &&
2965 config->num_pipes_active > 1)
2966 last_enabled_level = 0;
2967
2968 /* ILK: FBC WM must be disabled always */
2969 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
2970
2971 /* merge each WM1+ level */
2972 for (level = 1; level < num_levels; level++) {
2973 struct intel_wm_level *wm = &merged->wm[level];
2974
2975 ilk_merge_wm_level(dev_priv, level, wm);
2976
2977 if (level > last_enabled_level)
2978 wm->enable = false;
2979 else if (!ilk_validate_wm_level(level, max, wm))
2980 /* make sure all following levels get disabled */
2981 last_enabled_level = level - 1;
2982
2983 /*
2984 * The spec says it is preferred to disable
2985 * FBC WMs instead of disabling a WM level.
2986 */
2987 if (wm->fbc_val > max->fbc) {
2988 if (wm->enable)
2989 merged->fbc_wm_enabled = false;
2990 wm->fbc_val = 0;
2991 }
2992 }
2993
2994 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2995 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
2996 dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {
2997 for (level = 2; level < num_levels; level++) {
2998 struct intel_wm_level *wm = &merged->wm[level];
2999
3000 wm->enable = false;
3001 }
3002 }
3003}
3004
3005static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3006{
3007 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3008 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3009}
3010
3011/* The value we need to program into the WM_LPx latency field */
3012static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3013 int level)
3014{
3015 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3016 return 2 * level;
3017 else
3018 return dev_priv->display.wm.pri_latency[level];
3019}
3020
3021static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3022 const struct intel_pipe_wm *merged,
3023 enum intel_ddb_partitioning partitioning,
3024 struct ilk_wm_values *results)
3025{
3026 struct intel_crtc *crtc;
3027 int level, wm_lp;
3028
3029 results->enable_fbc_wm = merged->fbc_wm_enabled;
3030 results->partitioning = partitioning;
3031
3032 /* LP1+ register values */
3033 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3034 const struct intel_wm_level *r;
3035
3036 level = ilk_wm_lp_to_level(wm_lp, merged);
3037
3038 r = &merged->wm[level];
3039
3040 /*
3041 * Maintain the watermark values even if the level is
3042 * disabled. Doing otherwise could cause underruns.
3043 */
3044 results->wm_lp[wm_lp - 1] =
3045 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3046 WM_LP_PRIMARY(r->pri_val) |
3047 WM_LP_CURSOR(r->cur_val);
3048
3049 if (r->enable)
3050 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3051
3052 if (DISPLAY_VER(dev_priv) >= 8)
3053 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3054 else
3055 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3056
3057 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3058
3059 /*
3060 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3061 * level is disabled. Doing otherwise could cause underruns.
3062 */
3063 if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {
3064 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3065 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3066 }
3067 }
3068
3069 /* LP0 register values */
3070 for_each_intel_crtc(&dev_priv->drm, crtc) {
3071 enum pipe pipe = crtc->pipe;
3072 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3073 const struct intel_wm_level *r = &pipe_wm->wm[0];
3074
3075 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3076 continue;
3077
3078 results->wm_pipe[pipe] =
3079 WM0_PIPE_PRIMARY(r->pri_val) |
3080 WM0_PIPE_SPRITE(r->spr_val) |
3081 WM0_PIPE_CURSOR(r->cur_val);
3082 }
3083}
3084
3085/*
3086 * Find the result with the highest level enabled. Check for enable_fbc_wm in
3087 * case both are at the same level. Prefer r1 in case they're the same.
3088 */
3089static struct intel_pipe_wm *
3090ilk_find_best_result(struct drm_i915_private *dev_priv,
3091 struct intel_pipe_wm *r1,
3092 struct intel_pipe_wm *r2)
3093{
3094 int level, level1 = 0, level2 = 0;
3095
3096 for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3097 if (r1->wm[level].enable)
3098 level1 = level;
3099 if (r2->wm[level].enable)
3100 level2 = level;
3101 }
3102
3103 if (level1 == level2) {
3104 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3105 return r2;
3106 else
3107 return r1;
3108 } else if (level1 > level2) {
3109 return r1;
3110 } else {
3111 return r2;
3112 }
3113}
3114
3115/* dirty bits used to track which watermarks need changes */
3116#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3117#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3118#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3119#define WM_DIRTY_FBC (1 << 24)
3120#define WM_DIRTY_DDB (1 << 25)
3121
3122static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3123 const struct ilk_wm_values *old,
3124 const struct ilk_wm_values *new)
3125{
3126 unsigned int dirty = 0;
3127 enum pipe pipe;
3128 int wm_lp;
3129
3130 for_each_pipe(dev_priv, pipe) {
3131 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3132 dirty |= WM_DIRTY_PIPE(pipe);
3133 /* Must disable LP1+ watermarks too */
3134 dirty |= WM_DIRTY_LP_ALL;
3135 }
3136 }
3137
3138 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3139 dirty |= WM_DIRTY_FBC;
3140 /* Must disable LP1+ watermarks too */
3141 dirty |= WM_DIRTY_LP_ALL;
3142 }
3143
3144 if (old->partitioning != new->partitioning) {
3145 dirty |= WM_DIRTY_DDB;
3146 /* Must disable LP1+ watermarks too */
3147 dirty |= WM_DIRTY_LP_ALL;
3148 }
3149
3150 /* LP1+ watermarks already deemed dirty, no need to continue */
3151 if (dirty & WM_DIRTY_LP_ALL)
3152 return dirty;
3153
3154 /* Find the lowest numbered LP1+ watermark in need of an update... */
3155 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3156 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3157 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3158 break;
3159 }
3160
3161 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3162 for (; wm_lp <= 3; wm_lp++)
3163 dirty |= WM_DIRTY_LP(wm_lp);
3164
3165 return dirty;
3166}
3167
3168static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3169 unsigned int dirty)
3170{
3171 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3172 bool changed = false;
3173
3174 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3175 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3176 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3177 changed = true;
3178 }
3179 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3180 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3181 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3182 changed = true;
3183 }
3184 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3185 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3186 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3187 changed = true;
3188 }
3189
3190 /*
3191 * Don't touch WM_LP_SPRITE_ENABLE here.
3192 * Doing so could cause underruns.
3193 */
3194
3195 return changed;
3196}
3197
3198/*
3199 * The spec says we shouldn't write when we don't need, because every write
3200 * causes WMs to be re-evaluated, expending some power.
3201 */
3202static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3203 struct ilk_wm_values *results)
3204{
3205 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3206 unsigned int dirty;
3207
3208 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3209 if (!dirty)
3210 return;
3211
3212 _ilk_disable_lp_wm(dev_priv, dirty);
3213
3214 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3215 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3216 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3217 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3218 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3219 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3220
3221 if (dirty & WM_DIRTY_DDB) {
3222 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3223 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3224 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3225 WM_MISC_DATA_PARTITION_5_6);
3226 else
3227 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3228 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3229 DISP_DATA_PARTITION_5_6);
3230 }
3231
3232 if (dirty & WM_DIRTY_FBC)
3233 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3234 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3235
3236 if (dirty & WM_DIRTY_LP(1) &&
3237 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3238 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3239
3240 if (DISPLAY_VER(dev_priv) >= 7) {
3241 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3242 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3243 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3244 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3245 }
3246
3247 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3248 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3249 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3250 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3251 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3252 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3253
3254 dev_priv->display.wm.hw = *results;
3255}
3256
3257bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3258{
3259 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3260}
3261
3262static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3263 struct intel_wm_config *config)
3264{
3265 struct intel_crtc *crtc;
3266
3267 /* Compute the currently _active_ config */
3268 for_each_intel_crtc(&dev_priv->drm, crtc) {
3269 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3270
3271 if (!wm->pipe_enabled)
3272 continue;
3273
3274 config->sprites_enabled |= wm->sprites_enabled;
3275 config->sprites_scaled |= wm->sprites_scaled;
3276 config->num_pipes_active++;
3277 }
3278}
3279
3280static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3281{
3282 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3283 struct ilk_wm_maximums max;
3284 struct intel_wm_config config = {};
3285 struct ilk_wm_values results = {};
3286 enum intel_ddb_partitioning partitioning;
3287
3288 ilk_compute_wm_config(dev_priv, &config);
3289
3290 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3291 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3292
3293 /* 5/6 split only in single pipe config on IVB+ */
3294 if (DISPLAY_VER(dev_priv) >= 7 &&
3295 config.num_pipes_active == 1 && config.sprites_enabled) {
3296 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3297 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3298
3299 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3300 } else {
3301 best_lp_wm = &lp_wm_1_2;
3302 }
3303
3304 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3305 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3306
3307 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3308
3309 ilk_write_wm_values(dev_priv, &results);
3310}
3311
3312static void ilk_initial_watermarks(struct intel_atomic_state *state,
3313 struct intel_crtc *crtc)
3314{
3315 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3316 const struct intel_crtc_state *crtc_state =
3317 intel_atomic_get_new_crtc_state(state, crtc);
3318
3319 mutex_lock(&dev_priv->display.wm.wm_mutex);
3320 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3321 ilk_program_watermarks(dev_priv);
3322 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3323}
3324
3325static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3326 struct intel_crtc *crtc)
3327{
3328 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3329 const struct intel_crtc_state *crtc_state =
3330 intel_atomic_get_new_crtc_state(state, crtc);
3331
3332 if (!crtc_state->wm.need_postvbl_update)
3333 return;
3334
3335 mutex_lock(&dev_priv->display.wm.wm_mutex);
3336 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3337 ilk_program_watermarks(dev_priv);
3338 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3339}
3340
3341static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3342{
3343 struct drm_device *dev = crtc->base.dev;
3344 struct drm_i915_private *dev_priv = to_i915(dev);
3345 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3346 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3347 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3348 enum pipe pipe = crtc->pipe;
3349
3350 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3351
3352 memset(active, 0, sizeof(*active));
3353
3354 active->pipe_enabled = crtc->active;
3355
3356 if (active->pipe_enabled) {
3357 u32 tmp = hw->wm_pipe[pipe];
3358
3359 /*
3360 * For active pipes LP0 watermark is marked as
3361 * enabled, and LP1+ watermaks as disabled since
3362 * we can't really reverse compute them in case
3363 * multiple pipes are active.
3364 */
3365 active->wm[0].enable = true;
3366 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3367 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3368 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3369 } else {
3370 int level;
3371
3372 /*
3373 * For inactive pipes, all watermark levels
3374 * should be marked as enabled but zeroed,
3375 * which is what we'd compute them to.
3376 */
3377 for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3378 active->wm[level].enable = true;
3379 }
3380
3381 crtc->wm.active.ilk = *active;
3382}
3383
3384static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3385{
3386 struct drm_plane *plane;
3387 struct intel_crtc *crtc;
3388
3389 for_each_intel_crtc(state->dev, crtc) {
3390 struct intel_crtc_state *crtc_state;
3391
3392 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3393 if (IS_ERR(crtc_state))
3394 return PTR_ERR(crtc_state);
3395
3396 if (crtc_state->hw.active) {
3397 /*
3398 * Preserve the inherited flag to avoid
3399 * taking the full modeset path.
3400 */
3401 crtc_state->inherited = true;
3402 }
3403 }
3404
3405 drm_for_each_plane(plane, state->dev) {
3406 struct drm_plane_state *plane_state;
3407
3408 plane_state = drm_atomic_get_plane_state(state, plane);
3409 if (IS_ERR(plane_state))
3410 return PTR_ERR(plane_state);
3411 }
3412
3413 return 0;
3414}
3415
3416/*
3417 * Calculate what we think the watermarks should be for the state we've read
3418 * out of the hardware and then immediately program those watermarks so that
3419 * we ensure the hardware settings match our internal state.
3420 *
3421 * We can calculate what we think WM's should be by creating a duplicate of the
3422 * current state (which was constructed during hardware readout) and running it
3423 * through the atomic check code to calculate new watermark values in the
3424 * state object.
3425 */
3426void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3427{
3428 struct drm_atomic_state *state;
3429 struct intel_atomic_state *intel_state;
3430 struct intel_crtc *crtc;
3431 struct intel_crtc_state *crtc_state;
3432 struct drm_modeset_acquire_ctx ctx;
3433 int ret;
3434 int i;
3435
3436 /* Only supported on platforms that use atomic watermark design */
3437 if (!dev_priv->display.funcs.wm->optimize_watermarks)
3438 return;
3439
3440 if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3441 return;
3442
3443 state = drm_atomic_state_alloc(&dev_priv->drm);
3444 if (drm_WARN_ON(&dev_priv->drm, !state))
3445 return;
3446
3447 intel_state = to_intel_atomic_state(state);
3448
3449 drm_modeset_acquire_init(&ctx, 0);
3450
3451 state->acquire_ctx = &ctx;
3452 to_intel_atomic_state(state)->internal = true;
3453
3454retry:
3455 /*
3456 * Hardware readout is the only time we don't want to calculate
3457 * intermediate watermarks (since we don't trust the current
3458 * watermarks).
3459 */
3460 if (!HAS_GMCH(dev_priv))
3461 intel_state->skip_intermediate_wm = true;
3462
3463 ret = ilk_sanitize_watermarks_add_affected(state);
3464 if (ret)
3465 goto fail;
3466
3467 ret = intel_atomic_check(&dev_priv->drm, state);
3468 if (ret)
3469 goto fail;
3470
3471 /* Write calculated watermark values back */
3472 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3473 crtc_state->wm.need_postvbl_update = true;
3474 intel_optimize_watermarks(intel_state, crtc);
3475
3476 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3477 }
3478
3479fail:
3480 if (ret == -EDEADLK) {
3481 drm_atomic_state_clear(state);
3482 drm_modeset_backoff(&ctx);
3483 goto retry;
3484 }
3485
3486 /*
3487 * If we fail here, it means that the hardware appears to be
3488 * programmed in a way that shouldn't be possible, given our
3489 * understanding of watermark requirements. This might mean a
3490 * mistake in the hardware readout code or a mistake in the
3491 * watermark calculations for a given platform. Raise a WARN
3492 * so that this is noticeable.
3493 *
3494 * If this actually happens, we'll have to just leave the
3495 * BIOS-programmed watermarks untouched and hope for the best.
3496 */
3497 drm_WARN(&dev_priv->drm, ret,
3498 "Could not determine valid watermarks for inherited state\n");
3499
3500 drm_atomic_state_put(state);
3501
3502 drm_modeset_drop_locks(&ctx);
3503 drm_modeset_acquire_fini(&ctx);
3504}
3505
3506#define _FW_WM(value, plane) \
3507 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3508#define _FW_WM_VLV(value, plane) \
3509 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3510
3511static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3512 struct g4x_wm_values *wm)
3513{
3514 u32 tmp;
3515
3516 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3517 wm->sr.plane = _FW_WM(tmp, SR);
3518 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3519 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3520 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3521
3522 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3523 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3524 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3525 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3526 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3527 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3528 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3529
3530 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3531 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3532 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3533 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3534 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3535}
3536
3537static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3538 struct vlv_wm_values *wm)
3539{
3540 enum pipe pipe;
3541 u32 tmp;
3542
3543 for_each_pipe(dev_priv, pipe) {
3544 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3545
3546 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3547 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3548 wm->ddl[pipe].plane[PLANE_CURSOR] =
3549 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3550 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3551 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3552 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3553 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3554 }
3555
3556 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3557 wm->sr.plane = _FW_WM(tmp, SR);
3558 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3559 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3560 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3561
3562 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3563 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3564 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3565 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3566
3567 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3568 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3569
3570 if (IS_CHERRYVIEW(dev_priv)) {
3571 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3572 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3573 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3574
3575 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3576 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3577 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3578
3579 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3580 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3581 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3582
3583 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3584 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3585 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3586 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3587 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3588 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3589 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3590 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3591 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3592 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3593 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3594 } else {
3595 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3596 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3597 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3598
3599 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3600 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3601 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3602 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3603 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3604 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3605 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3606 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3607 }
3608}
3609
3610#undef _FW_WM
3611#undef _FW_WM_VLV
3612
3613static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3614{
3615 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3616 struct intel_crtc *crtc;
3617
3618 g4x_read_wm_values(dev_priv, wm);
3619
3620 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3621
3622 for_each_intel_crtc(&dev_priv->drm, crtc) {
3623 struct intel_crtc_state *crtc_state =
3624 to_intel_crtc_state(crtc->base.state);
3625 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3626 struct g4x_pipe_wm *raw;
3627 enum pipe pipe = crtc->pipe;
3628 enum plane_id plane_id;
3629 int level, max_level;
3630
3631 active->cxsr = wm->cxsr;
3632 active->hpll_en = wm->hpll_en;
3633 active->fbc_en = wm->fbc_en;
3634
3635 active->sr = wm->sr;
3636 active->hpll = wm->hpll;
3637
3638 for_each_plane_id_on_crtc(crtc, plane_id) {
3639 active->wm.plane[plane_id] =
3640 wm->pipe[pipe].plane[plane_id];
3641 }
3642
3643 if (wm->cxsr && wm->hpll_en)
3644 max_level = G4X_WM_LEVEL_HPLL;
3645 else if (wm->cxsr)
3646 max_level = G4X_WM_LEVEL_SR;
3647 else
3648 max_level = G4X_WM_LEVEL_NORMAL;
3649
3650 level = G4X_WM_LEVEL_NORMAL;
3651 raw = &crtc_state->wm.g4x.raw[level];
3652 for_each_plane_id_on_crtc(crtc, plane_id)
3653 raw->plane[plane_id] = active->wm.plane[plane_id];
3654
3655 level = G4X_WM_LEVEL_SR;
3656 if (level > max_level)
3657 goto out;
3658
3659 raw = &crtc_state->wm.g4x.raw[level];
3660 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3661 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3662 raw->plane[PLANE_SPRITE0] = 0;
3663 raw->fbc = active->sr.fbc;
3664
3665 level = G4X_WM_LEVEL_HPLL;
3666 if (level > max_level)
3667 goto out;
3668
3669 raw = &crtc_state->wm.g4x.raw[level];
3670 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3671 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3672 raw->plane[PLANE_SPRITE0] = 0;
3673 raw->fbc = active->hpll.fbc;
3674
3675 level++;
3676 out:
3677 for_each_plane_id_on_crtc(crtc, plane_id)
3678 g4x_raw_plane_wm_set(crtc_state, level,
3679 plane_id, USHRT_MAX);
3680 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3681
3682 g4x_invalidate_wms(crtc, active, level);
3683
3684 crtc_state->wm.g4x.optimal = *active;
3685 crtc_state->wm.g4x.intermediate = *active;
3686
3687 drm_dbg_kms(&dev_priv->drm,
3688 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3689 pipe_name(pipe),
3690 wm->pipe[pipe].plane[PLANE_PRIMARY],
3691 wm->pipe[pipe].plane[PLANE_CURSOR],
3692 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3693 }
3694
3695 drm_dbg_kms(&dev_priv->drm,
3696 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3697 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3698 drm_dbg_kms(&dev_priv->drm,
3699 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3700 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3701 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3702 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3703 str_yes_no(wm->fbc_en));
3704}
3705
3706static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3707{
3708 struct intel_plane *plane;
3709 struct intel_crtc *crtc;
3710
3711 mutex_lock(&dev_priv->display.wm.wm_mutex);
3712
3713 for_each_intel_plane(&dev_priv->drm, plane) {
3714 struct intel_crtc *crtc =
3715 intel_crtc_for_pipe(dev_priv, plane->pipe);
3716 struct intel_crtc_state *crtc_state =
3717 to_intel_crtc_state(crtc->base.state);
3718 struct intel_plane_state *plane_state =
3719 to_intel_plane_state(plane->base.state);
3720 enum plane_id plane_id = plane->id;
3721 int level;
3722
3723 if (plane_state->uapi.visible)
3724 continue;
3725
3726 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3727 struct g4x_pipe_wm *raw =
3728 &crtc_state->wm.g4x.raw[level];
3729
3730 raw->plane[plane_id] = 0;
3731
3732 if (plane_id == PLANE_PRIMARY)
3733 raw->fbc = 0;
3734 }
3735 }
3736
3737 for_each_intel_crtc(&dev_priv->drm, crtc) {
3738 struct intel_crtc_state *crtc_state =
3739 to_intel_crtc_state(crtc->base.state);
3740 int ret;
3741
3742 ret = _g4x_compute_pipe_wm(crtc_state);
3743 drm_WARN_ON(&dev_priv->drm, ret);
3744
3745 crtc_state->wm.g4x.intermediate =
3746 crtc_state->wm.g4x.optimal;
3747 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3748 }
3749
3750 g4x_program_watermarks(dev_priv);
3751
3752 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3753}
3754
3755static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3756{
3757 g4x_wm_get_hw_state(i915);
3758 g4x_wm_sanitize(i915);
3759}
3760
3761static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3762{
3763 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3764 struct intel_crtc *crtc;
3765 u32 val;
3766
3767 vlv_read_wm_values(dev_priv, wm);
3768
3769 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3770 wm->level = VLV_WM_LEVEL_PM2;
3771
3772 if (IS_CHERRYVIEW(dev_priv)) {
3773 vlv_punit_get(dev_priv);
3774
3775 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3776 if (val & DSP_MAXFIFO_PM5_ENABLE)
3777 wm->level = VLV_WM_LEVEL_PM5;
3778
3779 /*
3780 * If DDR DVFS is disabled in the BIOS, Punit
3781 * will never ack the request. So if that happens
3782 * assume we don't have to enable/disable DDR DVFS
3783 * dynamically. To test that just set the REQ_ACK
3784 * bit to poke the Punit, but don't change the
3785 * HIGH/LOW bits so that we don't actually change
3786 * the current state.
3787 */
3788 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3789 val |= FORCE_DDR_FREQ_REQ_ACK;
3790 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3791
3792 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3793 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3794 drm_dbg_kms(&dev_priv->drm,
3795 "Punit not acking DDR DVFS request, "
3796 "assuming DDR DVFS is disabled\n");
3797 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3798 } else {
3799 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3800 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3801 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3802 }
3803
3804 vlv_punit_put(dev_priv);
3805 }
3806
3807 for_each_intel_crtc(&dev_priv->drm, crtc) {
3808 struct intel_crtc_state *crtc_state =
3809 to_intel_crtc_state(crtc->base.state);
3810 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3811 const struct vlv_fifo_state *fifo_state =
3812 &crtc_state->wm.vlv.fifo_state;
3813 enum pipe pipe = crtc->pipe;
3814 enum plane_id plane_id;
3815 int level;
3816
3817 vlv_get_fifo_size(crtc_state);
3818
3819 active->num_levels = wm->level + 1;
3820 active->cxsr = wm->cxsr;
3821
3822 for (level = 0; level < active->num_levels; level++) {
3823 struct g4x_pipe_wm *raw =
3824 &crtc_state->wm.vlv.raw[level];
3825
3826 active->sr[level].plane = wm->sr.plane;
3827 active->sr[level].cursor = wm->sr.cursor;
3828
3829 for_each_plane_id_on_crtc(crtc, plane_id) {
3830 active->wm[level].plane[plane_id] =
3831 wm->pipe[pipe].plane[plane_id];
3832
3833 raw->plane[plane_id] =
3834 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3835 fifo_state->plane[plane_id]);
3836 }
3837 }
3838
3839 for_each_plane_id_on_crtc(crtc, plane_id)
3840 vlv_raw_plane_wm_set(crtc_state, level,
3841 plane_id, USHRT_MAX);
3842 vlv_invalidate_wms(crtc, active, level);
3843
3844 crtc_state->wm.vlv.optimal = *active;
3845 crtc_state->wm.vlv.intermediate = *active;
3846
3847 drm_dbg_kms(&dev_priv->drm,
3848 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3849 pipe_name(pipe),
3850 wm->pipe[pipe].plane[PLANE_PRIMARY],
3851 wm->pipe[pipe].plane[PLANE_CURSOR],
3852 wm->pipe[pipe].plane[PLANE_SPRITE0],
3853 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3854 }
3855
3856 drm_dbg_kms(&dev_priv->drm,
3857 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3858 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3859}
3860
3861static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3862{
3863 struct intel_plane *plane;
3864 struct intel_crtc *crtc;
3865
3866 mutex_lock(&dev_priv->display.wm.wm_mutex);
3867
3868 for_each_intel_plane(&dev_priv->drm, plane) {
3869 struct intel_crtc *crtc =
3870 intel_crtc_for_pipe(dev_priv, plane->pipe);
3871 struct intel_crtc_state *crtc_state =
3872 to_intel_crtc_state(crtc->base.state);
3873 struct intel_plane_state *plane_state =
3874 to_intel_plane_state(plane->base.state);
3875 enum plane_id plane_id = plane->id;
3876 int level;
3877
3878 if (plane_state->uapi.visible)
3879 continue;
3880
3881 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3882 struct g4x_pipe_wm *raw =
3883 &crtc_state->wm.vlv.raw[level];
3884
3885 raw->plane[plane_id] = 0;
3886 }
3887 }
3888
3889 for_each_intel_crtc(&dev_priv->drm, crtc) {
3890 struct intel_crtc_state *crtc_state =
3891 to_intel_crtc_state(crtc->base.state);
3892 int ret;
3893
3894 ret = _vlv_compute_pipe_wm(crtc_state);
3895 drm_WARN_ON(&dev_priv->drm, ret);
3896
3897 crtc_state->wm.vlv.intermediate =
3898 crtc_state->wm.vlv.optimal;
3899 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
3900 }
3901
3902 vlv_program_watermarks(dev_priv);
3903
3904 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3905}
3906
3907static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3908{
3909 vlv_wm_get_hw_state(i915);
3910 vlv_wm_sanitize(i915);
3911}
3912
3913/*
3914 * FIXME should probably kill this and improve
3915 * the real watermark readout/sanitation instead
3916 */
3917static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
3918{
3919 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
3920 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
3921 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
3922
3923 /*
3924 * Don't touch WM_LP_SPRITE_ENABLE here.
3925 * Doing so could cause underruns.
3926 */
3927}
3928
3929static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
3930{
3931 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3932 struct intel_crtc *crtc;
3933
3934 ilk_init_lp_watermarks(dev_priv);
3935
3936 for_each_intel_crtc(&dev_priv->drm, crtc)
3937 ilk_pipe_wm_get_hw_state(crtc);
3938
3939 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
3940 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
3941 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
3942
3943 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
3944 if (DISPLAY_VER(dev_priv) >= 7) {
3945 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
3946 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
3947 }
3948
3949 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3950 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
3951 WM_MISC_DATA_PARTITION_5_6) ?
3952 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3953 else if (IS_IVYBRIDGE(dev_priv))
3954 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
3955 DISP_DATA_PARTITION_5_6) ?
3956 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3957
3958 hw->enable_fbc_wm =
3959 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3960}
3961
3962static const struct intel_wm_funcs ilk_wm_funcs = {
3963 .compute_pipe_wm = ilk_compute_pipe_wm,
3964 .compute_intermediate_wm = ilk_compute_intermediate_wm,
3965 .initial_watermarks = ilk_initial_watermarks,
3966 .optimize_watermarks = ilk_optimize_watermarks,
3967 .get_hw_state = ilk_wm_get_hw_state,
3968};
3969
3970static const struct intel_wm_funcs vlv_wm_funcs = {
3971 .compute_pipe_wm = vlv_compute_pipe_wm,
3972 .compute_intermediate_wm = vlv_compute_intermediate_wm,
3973 .initial_watermarks = vlv_initial_watermarks,
3974 .optimize_watermarks = vlv_optimize_watermarks,
3975 .atomic_update_watermarks = vlv_atomic_update_fifo,
3976 .get_hw_state = vlv_wm_get_hw_state_and_sanitize,
3977};
3978
3979static const struct intel_wm_funcs g4x_wm_funcs = {
3980 .compute_pipe_wm = g4x_compute_pipe_wm,
3981 .compute_intermediate_wm = g4x_compute_intermediate_wm,
3982 .initial_watermarks = g4x_initial_watermarks,
3983 .optimize_watermarks = g4x_optimize_watermarks,
3984 .get_hw_state = g4x_wm_get_hw_state_and_sanitize,
3985};
3986
3987static const struct intel_wm_funcs pnv_wm_funcs = {
3988 .update_wm = pnv_update_wm,
3989};
3990
3991static const struct intel_wm_funcs i965_wm_funcs = {
3992 .update_wm = i965_update_wm,
3993};
3994
3995static const struct intel_wm_funcs i9xx_wm_funcs = {
3996 .update_wm = i9xx_update_wm,
3997};
3998
3999static const struct intel_wm_funcs i845_wm_funcs = {
4000 .update_wm = i845_update_wm,
4001};
4002
4003static const struct intel_wm_funcs nop_funcs = {
4004};
4005
4006void i9xx_wm_init(struct drm_i915_private *dev_priv)
4007{
4008 /* For FIFO watermark updates */
4009 if (HAS_PCH_SPLIT(dev_priv)) {
4010 ilk_setup_wm_latency(dev_priv);
4011 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4012 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4013 vlv_setup_wm_latency(dev_priv);
4014 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4015 } else if (IS_G4X(dev_priv)) {
4016 g4x_setup_wm_latency(dev_priv);
4017 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4018 } else if (IS_PINEVIEW(dev_priv)) {
4019 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4020 dev_priv->is_ddr3,
4021 dev_priv->fsb_freq,
4022 dev_priv->mem_freq)) {
4023 drm_info(&dev_priv->drm,
4024 "failed to find known CxSR latency "
4025 "(found ddr%s fsb freq %d, mem freq %d), "
4026 "disabling CxSR\n",
4027 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4028 dev_priv->fsb_freq, dev_priv->mem_freq);
4029 /* Disable CxSR and never update its watermark again */
4030 intel_set_memory_cxsr(dev_priv, false);
4031 dev_priv->display.funcs.wm = &nop_funcs;
4032 } else {
4033 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4034 }
4035 } else if (DISPLAY_VER(dev_priv) == 4) {
4036 dev_priv->display.funcs.wm = &i965_wm_funcs;
4037 } else if (DISPLAY_VER(dev_priv) == 3) {
4038 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4039 } else if (DISPLAY_VER(dev_priv) == 2) {
4040 if (INTEL_NUM_PIPES(dev_priv) == 1)
4041 dev_priv->display.funcs.wm = &i845_wm_funcs;
4042 else
4043 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4044 } else {
4045 drm_err(&dev_priv->drm,
4046 "unexpected fall-through in %s\n", __func__);
4047 dev_priv->display.funcs.wm = &nop_funcs;
4048 }
4049}