Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <linux/debugfs.h>
7
8#include <drm/drm_blend.h>
9
10#include "i915_drv.h"
11#include "i915_reg.h"
12#include "i9xx_wm.h"
13#include "intel_atomic.h"
14#include "intel_atomic_plane.h"
15#include "intel_bw.h"
16#include "intel_cdclk.h"
17#include "intel_crtc.h"
18#include "intel_cursor_regs.h"
19#include "intel_de.h"
20#include "intel_display.h"
21#include "intel_display_power.h"
22#include "intel_display_types.h"
23#include "intel_fb.h"
24#include "intel_fixed.h"
25#include "intel_pcode.h"
26#include "intel_wm.h"
27#include "skl_universal_plane_regs.h"
28#include "skl_watermark.h"
29#include "skl_watermark_regs.h"
30
31/*It is expected that DSB can do posted writes to every register in
32 * the pipe and planes within 100us. For flip queue use case, the
33 * recommended DSB execution time is 100us + one SAGV block time.
34 */
35#define DSB_EXE_TIME 100
36
37static void skl_sagv_disable(struct drm_i915_private *i915);
38
39/* Stores plane specific WM parameters */
40struct skl_wm_params {
41 bool x_tiled, y_tiled;
42 bool rc_surface;
43 bool is_planar;
44 u32 width;
45 u8 cpp;
46 u32 plane_pixel_rate;
47 u32 y_min_scanlines;
48 u32 plane_bytes_per_line;
49 uint_fixed_16_16_t plane_blocks_per_line;
50 uint_fixed_16_16_t y_tile_minimum;
51 u32 linetime_us;
52 u32 dbuf_block_size;
53};
54
55u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
56{
57 u8 enabled_slices = 0;
58 enum dbuf_slice slice;
59
60 for_each_dbuf_slice(i915, slice) {
61 if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
62 enabled_slices |= BIT(slice);
63 }
64
65 return enabled_slices;
66}
67
68/*
69 * FIXME: We still don't have the proper code detect if we need to apply the WA,
70 * so assume we'll always need it in order to avoid underruns.
71 */
72static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
73{
74 return DISPLAY_VER(i915) == 9;
75}
76
77bool
78intel_has_sagv(struct drm_i915_private *i915)
79{
80 return HAS_SAGV(i915) &&
81 i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
82}
83
84static u32
85intel_sagv_block_time(struct drm_i915_private *i915)
86{
87 if (DISPLAY_VER(i915) >= 14) {
88 u32 val;
89
90 val = intel_de_read(i915, MTL_LATENCY_SAGV);
91
92 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
93 } else if (DISPLAY_VER(i915) >= 12) {
94 u32 val = 0;
95 int ret;
96
97 ret = snb_pcode_read(&i915->uncore,
98 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
99 &val, NULL);
100 if (ret) {
101 drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
102 return 0;
103 }
104
105 return val;
106 } else if (DISPLAY_VER(i915) == 11) {
107 return 10;
108 } else if (HAS_SAGV(i915)) {
109 return 30;
110 } else {
111 return 0;
112 }
113}
114
115static void intel_sagv_init(struct drm_i915_private *i915)
116{
117 if (!HAS_SAGV(i915))
118 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
119
120 /*
121 * Probe to see if we have working SAGV control.
122 * For icl+ this was already determined by intel_bw_init_hw().
123 */
124 if (DISPLAY_VER(i915) < 11)
125 skl_sagv_disable(i915);
126
127 drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
128
129 i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
130
131 drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
132 str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
133
134 /* avoid overflow when adding with wm0 latency/etc. */
135 if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
136 "Excessive SAGV block time %u, ignoring\n",
137 i915->display.sagv.block_time_us))
138 i915->display.sagv.block_time_us = 0;
139
140 if (!intel_has_sagv(i915))
141 i915->display.sagv.block_time_us = 0;
142}
143
144/*
145 * SAGV dynamically adjusts the system agent voltage and clock frequencies
146 * depending on power and performance requirements. The display engine access
147 * to system memory is blocked during the adjustment time. Because of the
148 * blocking time, having this enabled can cause full system hangs and/or pipe
149 * underruns if we don't meet all of the following requirements:
150 *
151 * - <= 1 pipe enabled
152 * - All planes can enable watermarks for latencies >= SAGV engine block time
153 * - We're not using an interlaced display configuration
154 */
155static void skl_sagv_enable(struct drm_i915_private *i915)
156{
157 int ret;
158
159 if (!intel_has_sagv(i915))
160 return;
161
162 if (i915->display.sagv.status == I915_SAGV_ENABLED)
163 return;
164
165 drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
166 ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
167 GEN9_SAGV_ENABLE);
168
169 /* We don't need to wait for SAGV when enabling */
170
171 /*
172 * Some skl systems, pre-release machines in particular,
173 * don't actually have SAGV.
174 */
175 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
176 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
177 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
178 return;
179 } else if (ret < 0) {
180 drm_err(&i915->drm, "Failed to enable SAGV\n");
181 return;
182 }
183
184 i915->display.sagv.status = I915_SAGV_ENABLED;
185}
186
187static void skl_sagv_disable(struct drm_i915_private *i915)
188{
189 int ret;
190
191 if (!intel_has_sagv(i915))
192 return;
193
194 if (i915->display.sagv.status == I915_SAGV_DISABLED)
195 return;
196
197 drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
198 /* bspec says to keep retrying for at least 1 ms */
199 ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
200 GEN9_SAGV_DISABLE,
201 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
202 1);
203 /*
204 * Some skl systems, pre-release machines in particular,
205 * don't actually have SAGV.
206 */
207 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
208 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
209 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
210 return;
211 } else if (ret < 0) {
212 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
213 return;
214 }
215
216 i915->display.sagv.status = I915_SAGV_DISABLED;
217}
218
219static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
220{
221 struct drm_i915_private *i915 = to_i915(state->base.dev);
222 const struct intel_bw_state *new_bw_state =
223 intel_atomic_get_new_bw_state(state);
224
225 if (!new_bw_state)
226 return;
227
228 if (!intel_can_enable_sagv(i915, new_bw_state))
229 skl_sagv_disable(i915);
230}
231
232static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
233{
234 struct drm_i915_private *i915 = to_i915(state->base.dev);
235 const struct intel_bw_state *new_bw_state =
236 intel_atomic_get_new_bw_state(state);
237
238 if (!new_bw_state)
239 return;
240
241 if (intel_can_enable_sagv(i915, new_bw_state))
242 skl_sagv_enable(i915);
243}
244
245static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
246{
247 struct drm_i915_private *i915 = to_i915(state->base.dev);
248 const struct intel_bw_state *old_bw_state =
249 intel_atomic_get_old_bw_state(state);
250 const struct intel_bw_state *new_bw_state =
251 intel_atomic_get_new_bw_state(state);
252 u16 old_mask, new_mask;
253
254 if (!new_bw_state)
255 return;
256
257 old_mask = old_bw_state->qgv_points_mask;
258 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
259
260 if (old_mask == new_mask)
261 return;
262
263 WARN_ON(!new_bw_state->base.changed);
264
265 drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
266 old_mask, new_mask);
267
268 /*
269 * Restrict required qgv points before updating the configuration.
270 * According to BSpec we can't mask and unmask qgv points at the same
271 * time. Also masking should be done before updating the configuration
272 * and unmasking afterwards.
273 */
274 icl_pcode_restrict_qgv_points(i915, new_mask);
275}
276
277static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
278{
279 struct drm_i915_private *i915 = to_i915(state->base.dev);
280 const struct intel_bw_state *old_bw_state =
281 intel_atomic_get_old_bw_state(state);
282 const struct intel_bw_state *new_bw_state =
283 intel_atomic_get_new_bw_state(state);
284 u16 old_mask, new_mask;
285
286 if (!new_bw_state)
287 return;
288
289 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
290 new_mask = new_bw_state->qgv_points_mask;
291
292 if (old_mask == new_mask)
293 return;
294
295 WARN_ON(!new_bw_state->base.changed);
296
297 drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
298 old_mask, new_mask);
299
300 /*
301 * Allow required qgv points after updating the configuration.
302 * According to BSpec we can't mask and unmask qgv points at the same
303 * time. Also masking should be done before updating the configuration
304 * and unmasking afterwards.
305 */
306 icl_pcode_restrict_qgv_points(i915, new_mask);
307}
308
309void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
310{
311 struct drm_i915_private *i915 = to_i915(state->base.dev);
312
313 /*
314 * Just return if we can't control SAGV or don't have it.
315 * This is different from situation when we have SAGV but just can't
316 * afford it due to DBuf limitation - in case if SAGV is completely
317 * disabled in a BIOS, we are not even allowed to send a PCode request,
318 * as it will throw an error. So have to check it here.
319 */
320 if (!intel_has_sagv(i915))
321 return;
322
323 if (DISPLAY_VER(i915) >= 11)
324 icl_sagv_pre_plane_update(state);
325 else
326 skl_sagv_pre_plane_update(state);
327}
328
329void intel_sagv_post_plane_update(struct intel_atomic_state *state)
330{
331 struct drm_i915_private *i915 = to_i915(state->base.dev);
332
333 /*
334 * Just return if we can't control SAGV or don't have it.
335 * This is different from situation when we have SAGV but just can't
336 * afford it due to DBuf limitation - in case if SAGV is completely
337 * disabled in a BIOS, we are not even allowed to send a PCode request,
338 * as it will throw an error. So have to check it here.
339 */
340 if (!intel_has_sagv(i915))
341 return;
342
343 if (DISPLAY_VER(i915) >= 11)
344 icl_sagv_post_plane_update(state);
345 else
346 skl_sagv_post_plane_update(state);
347}
348
349static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
350{
351 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
352 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
353 enum plane_id plane_id;
354 int max_level = INT_MAX;
355
356 if (!intel_has_sagv(i915))
357 return false;
358
359 if (!crtc_state->hw.active)
360 return true;
361
362 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
363 return false;
364
365 for_each_plane_id_on_crtc(crtc, plane_id) {
366 const struct skl_plane_wm *wm =
367 &crtc_state->wm.skl.optimal.planes[plane_id];
368 int level;
369
370 /* Skip this plane if it's not enabled */
371 if (!wm->wm[0].enable)
372 continue;
373
374 /* Find the highest enabled wm level for this plane */
375 for (level = i915->display.wm.num_levels - 1;
376 !wm->wm[level].enable; --level)
377 { }
378
379 /* Highest common enabled wm level for all planes */
380 max_level = min(level, max_level);
381 }
382
383 /* No enabled planes? */
384 if (max_level == INT_MAX)
385 return true;
386
387 for_each_plane_id_on_crtc(crtc, plane_id) {
388 const struct skl_plane_wm *wm =
389 &crtc_state->wm.skl.optimal.planes[plane_id];
390
391 /*
392 * All enabled planes must have enabled a common wm level that
393 * can tolerate memory latencies higher than sagv_block_time_us
394 */
395 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
396 return false;
397 }
398
399 return true;
400}
401
402static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
403{
404 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
405 enum plane_id plane_id;
406
407 if (!crtc_state->hw.active)
408 return true;
409
410 for_each_plane_id_on_crtc(crtc, plane_id) {
411 const struct skl_plane_wm *wm =
412 &crtc_state->wm.skl.optimal.planes[plane_id];
413
414 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
415 return false;
416 }
417
418 return true;
419}
420
421static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
422{
423 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
424 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
425
426 if (!i915->display.params.enable_sagv)
427 return false;
428
429 if (DISPLAY_VER(i915) >= 12)
430 return tgl_crtc_can_enable_sagv(crtc_state);
431 else
432 return skl_crtc_can_enable_sagv(crtc_state);
433}
434
435bool intel_can_enable_sagv(struct drm_i915_private *i915,
436 const struct intel_bw_state *bw_state)
437{
438 if (DISPLAY_VER(i915) < 11 &&
439 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
440 return false;
441
442 return bw_state->pipe_sagv_reject == 0;
443}
444
445static int intel_compute_sagv_mask(struct intel_atomic_state *state)
446{
447 struct drm_i915_private *i915 = to_i915(state->base.dev);
448 int ret;
449 struct intel_crtc *crtc;
450 struct intel_crtc_state *new_crtc_state;
451 struct intel_bw_state *new_bw_state = NULL;
452 const struct intel_bw_state *old_bw_state = NULL;
453 int i;
454
455 for_each_new_intel_crtc_in_state(state, crtc,
456 new_crtc_state, i) {
457 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
458
459 new_bw_state = intel_atomic_get_bw_state(state);
460 if (IS_ERR(new_bw_state))
461 return PTR_ERR(new_bw_state);
462
463 old_bw_state = intel_atomic_get_old_bw_state(state);
464
465 /*
466 * We store use_sagv_wm in the crtc state rather than relying on
467 * that bw state since we have no convenient way to get at the
468 * latter from the plane commit hooks (especially in the legacy
469 * cursor case).
470 *
471 * drm_atomic_check_only() gets upset if we pull more crtcs
472 * into the state, so we have to calculate this based on the
473 * individual intel_crtc_can_enable_sagv() rather than
474 * the overall intel_can_enable_sagv(). Otherwise the
475 * crtcs not included in the commit would not switch to the
476 * SAGV watermarks when we are about to enable SAGV, and that
477 * would lead to underruns. This does mean extra power draw
478 * when only a subset of the crtcs are blocking SAGV as the
479 * other crtcs can't be allowed to use the more optimal
480 * normal (ie. non-SAGV) watermarks.
481 */
482 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
483 DISPLAY_VER(i915) >= 12 &&
484 intel_crtc_can_enable_sagv(new_crtc_state);
485
486 if (intel_crtc_can_enable_sagv(new_crtc_state))
487 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
488 else
489 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
490 }
491
492 if (!new_bw_state)
493 return 0;
494
495 new_bw_state->active_pipes =
496 intel_calc_active_pipes(state, old_bw_state->active_pipes);
497
498 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
499 ret = intel_atomic_lock_global_state(&new_bw_state->base);
500 if (ret)
501 return ret;
502 }
503
504 if (intel_can_enable_sagv(i915, new_bw_state) !=
505 intel_can_enable_sagv(i915, old_bw_state)) {
506 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
507 if (ret)
508 return ret;
509 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
510 ret = intel_atomic_lock_global_state(&new_bw_state->base);
511 if (ret)
512 return ret;
513 }
514
515 return 0;
516}
517
518static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
519 u16 start, u16 end)
520{
521 entry->start = start;
522 entry->end = end;
523
524 return end;
525}
526
527static int intel_dbuf_slice_size(struct drm_i915_private *i915)
528{
529 return DISPLAY_INFO(i915)->dbuf.size /
530 hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
531}
532
533static void
534skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
535 struct skl_ddb_entry *ddb)
536{
537 int slice_size = intel_dbuf_slice_size(i915);
538
539 if (!slice_mask) {
540 ddb->start = 0;
541 ddb->end = 0;
542 return;
543 }
544
545 ddb->start = (ffs(slice_mask) - 1) * slice_size;
546 ddb->end = fls(slice_mask) * slice_size;
547
548 WARN_ON(ddb->start >= ddb->end);
549 WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
550}
551
552static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
553{
554 struct skl_ddb_entry ddb;
555
556 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
557 slice_mask = BIT(DBUF_S1);
558 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
559 slice_mask = BIT(DBUF_S3);
560
561 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
562
563 return ddb.start;
564}
565
566u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
567 const struct skl_ddb_entry *entry)
568{
569 int slice_size = intel_dbuf_slice_size(i915);
570 enum dbuf_slice start_slice, end_slice;
571 u8 slice_mask = 0;
572
573 if (!skl_ddb_entry_size(entry))
574 return 0;
575
576 start_slice = entry->start / slice_size;
577 end_slice = (entry->end - 1) / slice_size;
578
579 /*
580 * Per plane DDB entry can in a really worst case be on multiple slices
581 * but single entry is anyway contigious.
582 */
583 while (start_slice <= end_slice) {
584 slice_mask |= BIT(start_slice);
585 start_slice++;
586 }
587
588 return slice_mask;
589}
590
591static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
592{
593 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
594 int hdisplay, vdisplay;
595
596 if (!crtc_state->hw.active)
597 return 0;
598
599 /*
600 * Watermark/ddb requirement highly depends upon width of the
601 * framebuffer, So instead of allocating DDB equally among pipes
602 * distribute DDB based on resolution/width of the display.
603 */
604 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
605
606 return hdisplay;
607}
608
609static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
610 enum pipe for_pipe,
611 unsigned int *weight_start,
612 unsigned int *weight_end,
613 unsigned int *weight_total)
614{
615 struct drm_i915_private *i915 =
616 to_i915(dbuf_state->base.state->base.dev);
617 enum pipe pipe;
618
619 *weight_start = 0;
620 *weight_end = 0;
621 *weight_total = 0;
622
623 for_each_pipe(i915, pipe) {
624 int weight = dbuf_state->weight[pipe];
625
626 /*
627 * Do not account pipes using other slice sets
628 * luckily as of current BSpec slice sets do not partially
629 * intersect(pipes share either same one slice or same slice set
630 * i.e no partial intersection), so it is enough to check for
631 * equality for now.
632 */
633 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
634 continue;
635
636 *weight_total += weight;
637 if (pipe < for_pipe) {
638 *weight_start += weight;
639 *weight_end += weight;
640 } else if (pipe == for_pipe) {
641 *weight_end += weight;
642 }
643 }
644}
645
646static int
647skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
648{
649 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
650 unsigned int weight_total, weight_start, weight_end;
651 const struct intel_dbuf_state *old_dbuf_state =
652 intel_atomic_get_old_dbuf_state(state);
653 struct intel_dbuf_state *new_dbuf_state =
654 intel_atomic_get_new_dbuf_state(state);
655 struct intel_crtc_state *crtc_state;
656 struct skl_ddb_entry ddb_slices;
657 enum pipe pipe = crtc->pipe;
658 unsigned int mbus_offset = 0;
659 u32 ddb_range_size;
660 u32 dbuf_slice_mask;
661 u32 start, end;
662 int ret;
663
664 if (new_dbuf_state->weight[pipe] == 0) {
665 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
666 goto out;
667 }
668
669 dbuf_slice_mask = new_dbuf_state->slices[pipe];
670
671 skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
672 mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
673 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
674
675 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
676 &weight_start, &weight_end, &weight_total);
677
678 start = ddb_range_size * weight_start / weight_total;
679 end = ddb_range_size * weight_end / weight_total;
680
681 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
682 ddb_slices.start - mbus_offset + start,
683 ddb_slices.start - mbus_offset + end);
684
685out:
686 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
687 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
688 &new_dbuf_state->ddb[pipe]))
689 return 0;
690
691 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
692 if (ret)
693 return ret;
694
695 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
696 if (IS_ERR(crtc_state))
697 return PTR_ERR(crtc_state);
698
699 /*
700 * Used for checking overlaps, so we need absolute
701 * offsets instead of MBUS relative offsets.
702 */
703 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
704 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
705
706 drm_dbg_kms(&i915->drm,
707 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
708 crtc->base.base.id, crtc->base.name,
709 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
710 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
711 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
712 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
713
714 return 0;
715}
716
717static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
718 int width, const struct drm_format_info *format,
719 u64 modifier, unsigned int rotation,
720 u32 plane_pixel_rate, struct skl_wm_params *wp,
721 int color_plane, unsigned int pan_x);
722
723static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
724 struct intel_plane *plane,
725 int level,
726 unsigned int latency,
727 const struct skl_wm_params *wp,
728 const struct skl_wm_level *result_prev,
729 struct skl_wm_level *result /* out */);
730
731static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
732 const struct skl_wm_params *wp)
733{
734 unsigned int latency = i915->display.wm.skl_latency[level];
735
736 if (latency == 0)
737 return 0;
738
739 /*
740 * WaIncreaseLatencyIPCEnabled: kbl,cfl
741 * Display WA #1141: kbl,cfl
742 */
743 if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
744 skl_watermark_ipc_enabled(i915))
745 latency += 4;
746
747 if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
748 latency += 15;
749
750 return latency;
751}
752
753static unsigned int
754skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
755 int num_active)
756{
757 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
758 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
759 struct skl_wm_level wm = {};
760 int ret, min_ddb_alloc = 0;
761 struct skl_wm_params wp;
762 int level;
763
764 ret = skl_compute_wm_params(crtc_state, 256,
765 drm_format_info(DRM_FORMAT_ARGB8888),
766 DRM_FORMAT_MOD_LINEAR,
767 DRM_MODE_ROTATE_0,
768 crtc_state->pixel_rate, &wp, 0, 0);
769 drm_WARN_ON(&i915->drm, ret);
770
771 for (level = 0; level < i915->display.wm.num_levels; level++) {
772 unsigned int latency = skl_wm_latency(i915, level, &wp);
773
774 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
775 if (wm.min_ddb_alloc == U16_MAX)
776 break;
777
778 min_ddb_alloc = wm.min_ddb_alloc;
779 }
780
781 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
782}
783
784static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
785{
786 skl_ddb_entry_init(entry,
787 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
788 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
789 if (entry->end)
790 entry->end++;
791}
792
793static void
794skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
795 const enum pipe pipe,
796 const enum plane_id plane_id,
797 struct skl_ddb_entry *ddb,
798 struct skl_ddb_entry *ddb_y)
799{
800 u32 val;
801
802 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
803 if (plane_id == PLANE_CURSOR) {
804 val = intel_de_read(i915, CUR_BUF_CFG(pipe));
805 skl_ddb_entry_init_from_hw(ddb, val);
806 return;
807 }
808
809 val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
810 skl_ddb_entry_init_from_hw(ddb, val);
811
812 if (DISPLAY_VER(i915) >= 11)
813 return;
814
815 val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
816 skl_ddb_entry_init_from_hw(ddb_y, val);
817}
818
819static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
820 struct skl_ddb_entry *ddb,
821 struct skl_ddb_entry *ddb_y)
822{
823 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
824 enum intel_display_power_domain power_domain;
825 enum pipe pipe = crtc->pipe;
826 intel_wakeref_t wakeref;
827 enum plane_id plane_id;
828
829 power_domain = POWER_DOMAIN_PIPE(pipe);
830 wakeref = intel_display_power_get_if_enabled(i915, power_domain);
831 if (!wakeref)
832 return;
833
834 for_each_plane_id_on_crtc(crtc, plane_id)
835 skl_ddb_get_hw_plane_state(i915, pipe,
836 plane_id,
837 &ddb[plane_id],
838 &ddb_y[plane_id]);
839
840 intel_display_power_put(i915, power_domain, wakeref);
841}
842
843struct dbuf_slice_conf_entry {
844 u8 active_pipes;
845 u8 dbuf_mask[I915_MAX_PIPES];
846 bool join_mbus;
847};
848
849/*
850 * Table taken from Bspec 12716
851 * Pipes do have some preferred DBuf slice affinity,
852 * plus there are some hardcoded requirements on how
853 * those should be distributed for multipipe scenarios.
854 * For more DBuf slices algorithm can get even more messy
855 * and less readable, so decided to use a table almost
856 * as is from BSpec itself - that way it is at least easier
857 * to compare, change and check.
858 */
859static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
860/* Autogenerated with igt/tools/intel_dbuf_map tool: */
861{
862 {
863 .active_pipes = BIT(PIPE_A),
864 .dbuf_mask = {
865 [PIPE_A] = BIT(DBUF_S1),
866 },
867 },
868 {
869 .active_pipes = BIT(PIPE_B),
870 .dbuf_mask = {
871 [PIPE_B] = BIT(DBUF_S1),
872 },
873 },
874 {
875 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
876 .dbuf_mask = {
877 [PIPE_A] = BIT(DBUF_S1),
878 [PIPE_B] = BIT(DBUF_S2),
879 },
880 },
881 {
882 .active_pipes = BIT(PIPE_C),
883 .dbuf_mask = {
884 [PIPE_C] = BIT(DBUF_S2),
885 },
886 },
887 {
888 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
889 .dbuf_mask = {
890 [PIPE_A] = BIT(DBUF_S1),
891 [PIPE_C] = BIT(DBUF_S2),
892 },
893 },
894 {
895 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
896 .dbuf_mask = {
897 [PIPE_B] = BIT(DBUF_S1),
898 [PIPE_C] = BIT(DBUF_S2),
899 },
900 },
901 {
902 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
903 .dbuf_mask = {
904 [PIPE_A] = BIT(DBUF_S1),
905 [PIPE_B] = BIT(DBUF_S1),
906 [PIPE_C] = BIT(DBUF_S2),
907 },
908 },
909 {}
910};
911
912/*
913 * Table taken from Bspec 49255
914 * Pipes do have some preferred DBuf slice affinity,
915 * plus there are some hardcoded requirements on how
916 * those should be distributed for multipipe scenarios.
917 * For more DBuf slices algorithm can get even more messy
918 * and less readable, so decided to use a table almost
919 * as is from BSpec itself - that way it is at least easier
920 * to compare, change and check.
921 */
922static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
923/* Autogenerated with igt/tools/intel_dbuf_map tool: */
924{
925 {
926 .active_pipes = BIT(PIPE_A),
927 .dbuf_mask = {
928 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
929 },
930 },
931 {
932 .active_pipes = BIT(PIPE_B),
933 .dbuf_mask = {
934 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
935 },
936 },
937 {
938 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
939 .dbuf_mask = {
940 [PIPE_A] = BIT(DBUF_S2),
941 [PIPE_B] = BIT(DBUF_S1),
942 },
943 },
944 {
945 .active_pipes = BIT(PIPE_C),
946 .dbuf_mask = {
947 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
948 },
949 },
950 {
951 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
952 .dbuf_mask = {
953 [PIPE_A] = BIT(DBUF_S1),
954 [PIPE_C] = BIT(DBUF_S2),
955 },
956 },
957 {
958 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
959 .dbuf_mask = {
960 [PIPE_B] = BIT(DBUF_S1),
961 [PIPE_C] = BIT(DBUF_S2),
962 },
963 },
964 {
965 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
966 .dbuf_mask = {
967 [PIPE_A] = BIT(DBUF_S1),
968 [PIPE_B] = BIT(DBUF_S1),
969 [PIPE_C] = BIT(DBUF_S2),
970 },
971 },
972 {
973 .active_pipes = BIT(PIPE_D),
974 .dbuf_mask = {
975 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
976 },
977 },
978 {
979 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
980 .dbuf_mask = {
981 [PIPE_A] = BIT(DBUF_S1),
982 [PIPE_D] = BIT(DBUF_S2),
983 },
984 },
985 {
986 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
987 .dbuf_mask = {
988 [PIPE_B] = BIT(DBUF_S1),
989 [PIPE_D] = BIT(DBUF_S2),
990 },
991 },
992 {
993 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
994 .dbuf_mask = {
995 [PIPE_A] = BIT(DBUF_S1),
996 [PIPE_B] = BIT(DBUF_S1),
997 [PIPE_D] = BIT(DBUF_S2),
998 },
999 },
1000 {
1001 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1002 .dbuf_mask = {
1003 [PIPE_C] = BIT(DBUF_S1),
1004 [PIPE_D] = BIT(DBUF_S2),
1005 },
1006 },
1007 {
1008 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1009 .dbuf_mask = {
1010 [PIPE_A] = BIT(DBUF_S1),
1011 [PIPE_C] = BIT(DBUF_S2),
1012 [PIPE_D] = BIT(DBUF_S2),
1013 },
1014 },
1015 {
1016 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1017 .dbuf_mask = {
1018 [PIPE_B] = BIT(DBUF_S1),
1019 [PIPE_C] = BIT(DBUF_S2),
1020 [PIPE_D] = BIT(DBUF_S2),
1021 },
1022 },
1023 {
1024 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1025 .dbuf_mask = {
1026 [PIPE_A] = BIT(DBUF_S1),
1027 [PIPE_B] = BIT(DBUF_S1),
1028 [PIPE_C] = BIT(DBUF_S2),
1029 [PIPE_D] = BIT(DBUF_S2),
1030 },
1031 },
1032 {}
1033};
1034
1035static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1036 {
1037 .active_pipes = BIT(PIPE_A),
1038 .dbuf_mask = {
1039 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1040 },
1041 },
1042 {
1043 .active_pipes = BIT(PIPE_B),
1044 .dbuf_mask = {
1045 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046 },
1047 },
1048 {
1049 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1050 .dbuf_mask = {
1051 [PIPE_A] = BIT(DBUF_S1),
1052 [PIPE_B] = BIT(DBUF_S2),
1053 },
1054 },
1055 {
1056 .active_pipes = BIT(PIPE_C),
1057 .dbuf_mask = {
1058 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1059 },
1060 },
1061 {
1062 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1063 .dbuf_mask = {
1064 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1065 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1066 },
1067 },
1068 {
1069 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1070 .dbuf_mask = {
1071 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1072 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1073 },
1074 },
1075 {
1076 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1077 .dbuf_mask = {
1078 [PIPE_A] = BIT(DBUF_S1),
1079 [PIPE_B] = BIT(DBUF_S2),
1080 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1081 },
1082 },
1083 {
1084 .active_pipes = BIT(PIPE_D),
1085 .dbuf_mask = {
1086 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1087 },
1088 },
1089 {
1090 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1091 .dbuf_mask = {
1092 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1093 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1094 },
1095 },
1096 {
1097 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1098 .dbuf_mask = {
1099 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1100 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1101 },
1102 },
1103 {
1104 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1105 .dbuf_mask = {
1106 [PIPE_A] = BIT(DBUF_S1),
1107 [PIPE_B] = BIT(DBUF_S2),
1108 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1109 },
1110 },
1111 {
1112 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1113 .dbuf_mask = {
1114 [PIPE_C] = BIT(DBUF_S3),
1115 [PIPE_D] = BIT(DBUF_S4),
1116 },
1117 },
1118 {
1119 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1120 .dbuf_mask = {
1121 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1122 [PIPE_C] = BIT(DBUF_S3),
1123 [PIPE_D] = BIT(DBUF_S4),
1124 },
1125 },
1126 {
1127 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1128 .dbuf_mask = {
1129 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1130 [PIPE_C] = BIT(DBUF_S3),
1131 [PIPE_D] = BIT(DBUF_S4),
1132 },
1133 },
1134 {
1135 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1136 .dbuf_mask = {
1137 [PIPE_A] = BIT(DBUF_S1),
1138 [PIPE_B] = BIT(DBUF_S2),
1139 [PIPE_C] = BIT(DBUF_S3),
1140 [PIPE_D] = BIT(DBUF_S4),
1141 },
1142 },
1143 {}
1144};
1145
1146static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1147 /*
1148 * Keep the join_mbus cases first so check_mbus_joined()
1149 * will prefer them over the !join_mbus cases.
1150 */
1151 {
1152 .active_pipes = BIT(PIPE_A),
1153 .dbuf_mask = {
1154 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1155 },
1156 .join_mbus = true,
1157 },
1158 {
1159 .active_pipes = BIT(PIPE_B),
1160 .dbuf_mask = {
1161 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1162 },
1163 .join_mbus = true,
1164 },
1165 {
1166 .active_pipes = BIT(PIPE_A),
1167 .dbuf_mask = {
1168 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1169 },
1170 .join_mbus = false,
1171 },
1172 {
1173 .active_pipes = BIT(PIPE_B),
1174 .dbuf_mask = {
1175 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1176 },
1177 .join_mbus = false,
1178 },
1179 {
1180 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1181 .dbuf_mask = {
1182 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1183 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184 },
1185 },
1186 {
1187 .active_pipes = BIT(PIPE_C),
1188 .dbuf_mask = {
1189 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1190 },
1191 },
1192 {
1193 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1194 .dbuf_mask = {
1195 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1196 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1197 },
1198 },
1199 {
1200 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1201 .dbuf_mask = {
1202 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1203 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1204 },
1205 },
1206 {
1207 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1208 .dbuf_mask = {
1209 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1210 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1211 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212 },
1213 },
1214 {
1215 .active_pipes = BIT(PIPE_D),
1216 .dbuf_mask = {
1217 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1218 },
1219 },
1220 {
1221 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1222 .dbuf_mask = {
1223 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1224 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1225 },
1226 },
1227 {
1228 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1229 .dbuf_mask = {
1230 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1231 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1232 },
1233 },
1234 {
1235 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1236 .dbuf_mask = {
1237 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1238 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1239 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1240 },
1241 },
1242 {
1243 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1244 .dbuf_mask = {
1245 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1246 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1247 },
1248 },
1249 {
1250 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1251 .dbuf_mask = {
1252 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1254 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1255 },
1256 },
1257 {
1258 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1259 .dbuf_mask = {
1260 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1261 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1262 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1263 },
1264 },
1265 {
1266 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1267 .dbuf_mask = {
1268 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1269 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1270 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1271 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1272 },
1273 },
1274 {}
1275
1276};
1277
1278static bool check_mbus_joined(u8 active_pipes,
1279 const struct dbuf_slice_conf_entry *dbuf_slices)
1280{
1281 int i;
1282
1283 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1284 if (dbuf_slices[i].active_pipes == active_pipes)
1285 return dbuf_slices[i].join_mbus;
1286 }
1287 return false;
1288}
1289
1290static bool adlp_check_mbus_joined(u8 active_pipes)
1291{
1292 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1293}
1294
1295static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1296 const struct dbuf_slice_conf_entry *dbuf_slices)
1297{
1298 int i;
1299
1300 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1301 if (dbuf_slices[i].active_pipes == active_pipes &&
1302 dbuf_slices[i].join_mbus == join_mbus)
1303 return dbuf_slices[i].dbuf_mask[pipe];
1304 }
1305 return 0;
1306}
1307
1308/*
1309 * This function finds an entry with same enabled pipe configuration and
1310 * returns correspondent DBuf slice mask as stated in BSpec for particular
1311 * platform.
1312 */
1313static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1314{
1315 /*
1316 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1317 * required calculating "pipe ratio" in order to determine
1318 * if one or two slices can be used for single pipe configurations
1319 * as additional constraint to the existing table.
1320 * However based on recent info, it should be not "pipe ratio"
1321 * but rather ratio between pixel_rate and cdclk with additional
1322 * constants, so for now we are using only table until this is
1323 * clarified. Also this is the reason why crtc_state param is
1324 * still here - we will need it once those additional constraints
1325 * pop up.
1326 */
1327 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1328 icl_allowed_dbufs);
1329}
1330
1331static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1332{
1333 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1334 tgl_allowed_dbufs);
1335}
1336
1337static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1338{
1339 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1340 adlp_allowed_dbufs);
1341}
1342
1343static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1344{
1345 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1346 dg2_allowed_dbufs);
1347}
1348
1349static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1350{
1351 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1352 enum pipe pipe = crtc->pipe;
1353
1354 if (IS_DG2(i915))
1355 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1356 else if (DISPLAY_VER(i915) >= 13)
1357 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1358 else if (DISPLAY_VER(i915) == 12)
1359 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1360 else if (DISPLAY_VER(i915) == 11)
1361 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1362 /*
1363 * For anything else just return one slice yet.
1364 * Should be extended for other platforms.
1365 */
1366 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1367}
1368
1369static bool
1370use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1371 struct intel_plane *plane)
1372{
1373 struct drm_i915_private *i915 = to_i915(plane->base.dev);
1374
1375 return DISPLAY_VER(i915) >= 13 &&
1376 crtc_state->uapi.async_flip &&
1377 plane->async_flip;
1378}
1379
1380static u64
1381skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1382{
1383 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1384 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1385 enum plane_id plane_id;
1386 u64 data_rate = 0;
1387
1388 for_each_plane_id_on_crtc(crtc, plane_id) {
1389 if (plane_id == PLANE_CURSOR)
1390 continue;
1391
1392 data_rate += crtc_state->rel_data_rate[plane_id];
1393
1394 if (DISPLAY_VER(i915) < 11)
1395 data_rate += crtc_state->rel_data_rate_y[plane_id];
1396 }
1397
1398 return data_rate;
1399}
1400
1401const struct skl_wm_level *
1402skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1403 enum plane_id plane_id,
1404 int level)
1405{
1406 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1407
1408 if (level == 0 && pipe_wm->use_sagv_wm)
1409 return &wm->sagv.wm0;
1410
1411 return &wm->wm[level];
1412}
1413
1414const struct skl_wm_level *
1415skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1416 enum plane_id plane_id)
1417{
1418 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1419
1420 if (pipe_wm->use_sagv_wm)
1421 return &wm->sagv.trans_wm;
1422
1423 return &wm->trans_wm;
1424}
1425
1426/*
1427 * We only disable the watermarks for each plane if
1428 * they exceed the ddb allocation of said plane. This
1429 * is done so that we don't end up touching cursor
1430 * watermarks needlessly when some other plane reduces
1431 * our max possible watermark level.
1432 *
1433 * Bspec has this to say about the PLANE_WM enable bit:
1434 * "All the watermarks at this level for all enabled
1435 * planes must be enabled before the level will be used."
1436 * So this is actually safe to do.
1437 */
1438static void
1439skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1440{
1441 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1442 memset(wm, 0, sizeof(*wm));
1443}
1444
1445static void
1446skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1447 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1448{
1449 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1450 uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1451 memset(wm, 0, sizeof(*wm));
1452 memset(uv_wm, 0, sizeof(*uv_wm));
1453 }
1454}
1455
1456static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1457 const struct skl_plane_wm *wm)
1458{
1459 /*
1460 * Wa_1408961008:icl, ehl
1461 * Wa_14012656716:tgl, adl
1462 * Wa_14017887344:icl
1463 * Wa_14017868169:adl, tgl
1464 * Due to some power saving optimizations, different subsystems
1465 * like PSR, might still use even disabled wm level registers,
1466 * for "reference", so lets keep at least the values sane.
1467 * Considering amount of WA requiring us to do similar things, was
1468 * decided to simply do it for all of the platforms, as those wm
1469 * levels are disabled, this isn't going to do harm anyway.
1470 */
1471 return level > 0 && !wm->wm[level].enable;
1472}
1473
1474struct skl_plane_ddb_iter {
1475 u64 data_rate;
1476 u16 start, size;
1477};
1478
1479static void
1480skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1481 struct skl_ddb_entry *ddb,
1482 const struct skl_wm_level *wm,
1483 u64 data_rate)
1484{
1485 u16 size, extra = 0;
1486
1487 if (data_rate) {
1488 extra = min_t(u16, iter->size,
1489 DIV64_U64_ROUND_UP(iter->size * data_rate,
1490 iter->data_rate));
1491 iter->size -= extra;
1492 iter->data_rate -= data_rate;
1493 }
1494
1495 /*
1496 * Keep ddb entry of all disabled planes explicitly zeroed
1497 * to avoid skl_ddb_add_affected_planes() adding them to
1498 * the state when other planes change their allocations.
1499 */
1500 size = wm->min_ddb_alloc + extra;
1501 if (size)
1502 iter->start = skl_ddb_entry_init(ddb, iter->start,
1503 iter->start + size);
1504}
1505
1506static int
1507skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1508 struct intel_crtc *crtc)
1509{
1510 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1511 struct intel_crtc_state *crtc_state =
1512 intel_atomic_get_new_crtc_state(state, crtc);
1513 const struct intel_dbuf_state *dbuf_state =
1514 intel_atomic_get_new_dbuf_state(state);
1515 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1516 int num_active = hweight8(dbuf_state->active_pipes);
1517 struct skl_plane_ddb_iter iter;
1518 enum plane_id plane_id;
1519 u16 cursor_size;
1520 u32 blocks;
1521 int level;
1522
1523 /* Clear the partitioning for disabled planes. */
1524 memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1525 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1526
1527 if (!crtc_state->hw.active)
1528 return 0;
1529
1530 iter.start = alloc->start;
1531 iter.size = skl_ddb_entry_size(alloc);
1532 if (iter.size == 0)
1533 return 0;
1534
1535 /* Allocate fixed number of blocks for cursor. */
1536 cursor_size = skl_cursor_allocation(crtc_state, num_active);
1537 iter.size -= cursor_size;
1538 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1539 alloc->end - cursor_size, alloc->end);
1540
1541 iter.data_rate = skl_total_relative_data_rate(crtc_state);
1542
1543 /*
1544 * Find the highest watermark level for which we can satisfy the block
1545 * requirement of active planes.
1546 */
1547 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1548 blocks = 0;
1549 for_each_plane_id_on_crtc(crtc, plane_id) {
1550 const struct skl_plane_wm *wm =
1551 &crtc_state->wm.skl.optimal.planes[plane_id];
1552
1553 if (plane_id == PLANE_CURSOR) {
1554 const struct skl_ddb_entry *ddb =
1555 &crtc_state->wm.skl.plane_ddb[plane_id];
1556
1557 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1558 drm_WARN_ON(&i915->drm,
1559 wm->wm[level].min_ddb_alloc != U16_MAX);
1560 blocks = U32_MAX;
1561 break;
1562 }
1563 continue;
1564 }
1565
1566 blocks += wm->wm[level].min_ddb_alloc;
1567 blocks += wm->uv_wm[level].min_ddb_alloc;
1568 }
1569
1570 if (blocks <= iter.size) {
1571 iter.size -= blocks;
1572 break;
1573 }
1574 }
1575
1576 if (level < 0) {
1577 drm_dbg_kms(&i915->drm,
1578 "Requested display configuration exceeds system DDB limitations");
1579 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1580 blocks, iter.size);
1581 return -EINVAL;
1582 }
1583
1584 /* avoid the WARN later when we don't allocate any extra DDB */
1585 if (iter.data_rate == 0)
1586 iter.size = 0;
1587
1588 /*
1589 * Grant each plane the blocks it requires at the highest achievable
1590 * watermark level, plus an extra share of the leftover blocks
1591 * proportional to its relative data rate.
1592 */
1593 for_each_plane_id_on_crtc(crtc, plane_id) {
1594 struct skl_ddb_entry *ddb =
1595 &crtc_state->wm.skl.plane_ddb[plane_id];
1596 struct skl_ddb_entry *ddb_y =
1597 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1598 const struct skl_plane_wm *wm =
1599 &crtc_state->wm.skl.optimal.planes[plane_id];
1600
1601 if (plane_id == PLANE_CURSOR)
1602 continue;
1603
1604 if (DISPLAY_VER(i915) < 11 &&
1605 crtc_state->nv12_planes & BIT(plane_id)) {
1606 skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1607 crtc_state->rel_data_rate_y[plane_id]);
1608 skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1609 crtc_state->rel_data_rate[plane_id]);
1610 } else {
1611 skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1612 crtc_state->rel_data_rate[plane_id]);
1613 }
1614 }
1615 drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1616
1617 /*
1618 * When we calculated watermark values we didn't know how high
1619 * of a level we'd actually be able to hit, so we just marked
1620 * all levels as "enabled." Go back now and disable the ones
1621 * that aren't actually possible.
1622 */
1623 for (level++; level < i915->display.wm.num_levels; level++) {
1624 for_each_plane_id_on_crtc(crtc, plane_id) {
1625 const struct skl_ddb_entry *ddb =
1626 &crtc_state->wm.skl.plane_ddb[plane_id];
1627 const struct skl_ddb_entry *ddb_y =
1628 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1629 struct skl_plane_wm *wm =
1630 &crtc_state->wm.skl.optimal.planes[plane_id];
1631
1632 if (DISPLAY_VER(i915) < 11 &&
1633 crtc_state->nv12_planes & BIT(plane_id))
1634 skl_check_nv12_wm_level(&wm->wm[level],
1635 &wm->uv_wm[level],
1636 ddb_y, ddb);
1637 else
1638 skl_check_wm_level(&wm->wm[level], ddb);
1639
1640 if (skl_need_wm_copy_wa(i915, level, wm)) {
1641 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1642 wm->wm[level].lines = wm->wm[level - 1].lines;
1643 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1644 }
1645 }
1646 }
1647
1648 /*
1649 * Go back and disable the transition and SAGV watermarks
1650 * if it turns out we don't have enough DDB blocks for them.
1651 */
1652 for_each_plane_id_on_crtc(crtc, plane_id) {
1653 const struct skl_ddb_entry *ddb =
1654 &crtc_state->wm.skl.plane_ddb[plane_id];
1655 const struct skl_ddb_entry *ddb_y =
1656 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1657 struct skl_plane_wm *wm =
1658 &crtc_state->wm.skl.optimal.planes[plane_id];
1659
1660 if (DISPLAY_VER(i915) < 11 &&
1661 crtc_state->nv12_planes & BIT(plane_id)) {
1662 skl_check_wm_level(&wm->trans_wm, ddb_y);
1663 } else {
1664 WARN_ON(skl_ddb_entry_size(ddb_y));
1665
1666 skl_check_wm_level(&wm->trans_wm, ddb);
1667 }
1668
1669 skl_check_wm_level(&wm->sagv.wm0, ddb);
1670 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1671 }
1672
1673 return 0;
1674}
1675
1676/*
1677 * The max latency should be 257 (max the punit can code is 255 and we add 2us
1678 * for the read latency) and cpp should always be <= 8, so that
1679 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1680 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1681 */
1682static uint_fixed_16_16_t
1683skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1684 u8 cpp, u32 latency, u32 dbuf_block_size)
1685{
1686 u32 wm_intermediate_val;
1687 uint_fixed_16_16_t ret;
1688
1689 if (latency == 0)
1690 return FP_16_16_MAX;
1691
1692 wm_intermediate_val = latency * pixel_rate * cpp;
1693 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1694
1695 if (DISPLAY_VER(i915) >= 10)
1696 ret = add_fixed16_u32(ret, 1);
1697
1698 return ret;
1699}
1700
1701static uint_fixed_16_16_t
1702skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1703 uint_fixed_16_16_t plane_blocks_per_line)
1704{
1705 u32 wm_intermediate_val;
1706 uint_fixed_16_16_t ret;
1707
1708 if (latency == 0)
1709 return FP_16_16_MAX;
1710
1711 wm_intermediate_val = latency * pixel_rate;
1712 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1713 pipe_htotal * 1000);
1714 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1715 return ret;
1716}
1717
1718static uint_fixed_16_16_t
1719intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1720{
1721 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1722 u32 pixel_rate;
1723 u32 crtc_htotal;
1724 uint_fixed_16_16_t linetime_us;
1725
1726 if (!crtc_state->hw.active)
1727 return u32_to_fixed16(0);
1728
1729 pixel_rate = crtc_state->pixel_rate;
1730
1731 if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1732 return u32_to_fixed16(0);
1733
1734 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1735 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1736
1737 return linetime_us;
1738}
1739
1740static int
1741skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1742 int width, const struct drm_format_info *format,
1743 u64 modifier, unsigned int rotation,
1744 u32 plane_pixel_rate, struct skl_wm_params *wp,
1745 int color_plane, unsigned int pan_x)
1746{
1747 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1748 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1749 u32 interm_pbpl;
1750
1751 /* only planar format has two planes */
1752 if (color_plane == 1 &&
1753 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1754 drm_dbg_kms(&i915->drm,
1755 "Non planar format have single plane\n");
1756 return -EINVAL;
1757 }
1758
1759 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1760 wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1761 intel_fb_is_tiled_modifier(modifier);
1762 wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1763 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1764
1765 wp->width = width;
1766 if (color_plane == 1 && wp->is_planar)
1767 wp->width /= 2;
1768
1769 wp->cpp = format->cpp[color_plane];
1770 wp->plane_pixel_rate = plane_pixel_rate;
1771
1772 if (DISPLAY_VER(i915) >= 11 &&
1773 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1774 wp->dbuf_block_size = 256;
1775 else
1776 wp->dbuf_block_size = 512;
1777
1778 if (drm_rotation_90_or_270(rotation)) {
1779 switch (wp->cpp) {
1780 case 1:
1781 wp->y_min_scanlines = 16;
1782 break;
1783 case 2:
1784 wp->y_min_scanlines = 8;
1785 break;
1786 case 4:
1787 wp->y_min_scanlines = 4;
1788 break;
1789 default:
1790 MISSING_CASE(wp->cpp);
1791 return -EINVAL;
1792 }
1793 } else {
1794 wp->y_min_scanlines = 4;
1795 }
1796
1797 if (skl_needs_memory_bw_wa(i915))
1798 wp->y_min_scanlines *= 2;
1799
1800 wp->plane_bytes_per_line = wp->width * wp->cpp;
1801 if (wp->y_tiled) {
1802 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1803 wp->y_min_scanlines,
1804 wp->dbuf_block_size);
1805
1806 if (DISPLAY_VER(i915) >= 30)
1807 interm_pbpl += (pan_x != 0);
1808 else if (DISPLAY_VER(i915) >= 10)
1809 interm_pbpl++;
1810
1811 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1812 wp->y_min_scanlines);
1813 } else {
1814 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1815 wp->dbuf_block_size);
1816
1817 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1818 interm_pbpl++;
1819
1820 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1821 }
1822
1823 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1824 wp->plane_blocks_per_line);
1825
1826 wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1827
1828 return 0;
1829}
1830
1831static int
1832skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1833 const struct intel_plane_state *plane_state,
1834 struct skl_wm_params *wp, int color_plane)
1835{
1836 const struct drm_framebuffer *fb = plane_state->hw.fb;
1837 int width;
1838
1839 /*
1840 * Src coordinates are already rotated by 270 degrees for
1841 * the 90/270 degree plane rotation cases (to match the
1842 * GTT mapping), hence no need to account for rotation here.
1843 */
1844 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1845
1846 return skl_compute_wm_params(crtc_state, width,
1847 fb->format, fb->modifier,
1848 plane_state->hw.rotation,
1849 intel_plane_pixel_rate(crtc_state, plane_state),
1850 wp, color_plane,
1851 plane_state->uapi.src.x1);
1852}
1853
1854static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1855{
1856 if (DISPLAY_VER(i915) >= 10)
1857 return true;
1858
1859 /* The number of lines are ignored for the level 0 watermark. */
1860 return level > 0;
1861}
1862
1863static int skl_wm_max_lines(struct drm_i915_private *i915)
1864{
1865 if (DISPLAY_VER(i915) >= 13)
1866 return 255;
1867 else
1868 return 31;
1869}
1870
1871static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1872 struct intel_plane *plane,
1873 int level,
1874 unsigned int latency,
1875 const struct skl_wm_params *wp,
1876 const struct skl_wm_level *result_prev,
1877 struct skl_wm_level *result /* out */)
1878{
1879 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1880 uint_fixed_16_16_t method1, method2;
1881 uint_fixed_16_16_t selected_result;
1882 u32 blocks, lines, min_ddb_alloc = 0;
1883
1884 if (latency == 0 ||
1885 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1886 /* reject it */
1887 result->min_ddb_alloc = U16_MAX;
1888 return;
1889 }
1890
1891 method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1892 wp->cpp, latency, wp->dbuf_block_size);
1893 method2 = skl_wm_method2(wp->plane_pixel_rate,
1894 crtc_state->hw.pipe_mode.crtc_htotal,
1895 latency,
1896 wp->plane_blocks_per_line);
1897
1898 if (wp->y_tiled) {
1899 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1900 } else {
1901 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1902 wp->dbuf_block_size < 1) &&
1903 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1904 selected_result = method2;
1905 } else if (latency >= wp->linetime_us) {
1906 if (DISPLAY_VER(i915) == 9)
1907 selected_result = min_fixed16(method1, method2);
1908 else
1909 selected_result = method2;
1910 } else {
1911 selected_result = method1;
1912 }
1913 }
1914
1915 blocks = fixed16_to_u32_round_up(selected_result);
1916 if (DISPLAY_VER(i915) < 30)
1917 blocks++;
1918
1919 /*
1920 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1921 * as there will be at minimum one line for lines configuration. This
1922 * is a work around for FIFO underruns observed with resolutions like
1923 * 4k 60 Hz in single channel DRAM configurations.
1924 *
1925 * As per the Bspec 49325, if the ddb allocation can hold at least
1926 * one plane_blocks_per_line, we should have selected method2 in
1927 * the above logic. Assuming that modern versions have enough dbuf
1928 * and method2 guarantees blocks equivalent to at least 1 line,
1929 * select the blocks as plane_blocks_per_line.
1930 *
1931 * TODO: Revisit the logic when we have better understanding on DRAM
1932 * channels' impact on the level 0 memory latency and the relevant
1933 * wm calculations.
1934 */
1935 if (skl_wm_has_lines(i915, level))
1936 blocks = max(blocks,
1937 fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1938 lines = div_round_up_fixed16(selected_result,
1939 wp->plane_blocks_per_line);
1940
1941 if (DISPLAY_VER(i915) == 9) {
1942 /* Display WA #1125: skl,bxt,kbl */
1943 if (level == 0 && wp->rc_surface)
1944 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1945
1946 /* Display WA #1126: skl,bxt,kbl */
1947 if (level >= 1 && level <= 7) {
1948 if (wp->y_tiled) {
1949 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1950 lines += wp->y_min_scanlines;
1951 } else {
1952 blocks++;
1953 }
1954
1955 /*
1956 * Make sure result blocks for higher latency levels are
1957 * at least as high as level below the current level.
1958 * Assumption in DDB algorithm optimization for special
1959 * cases. Also covers Display WA #1125 for RC.
1960 */
1961 if (result_prev->blocks > blocks)
1962 blocks = result_prev->blocks;
1963 }
1964 }
1965
1966 if (DISPLAY_VER(i915) >= 11) {
1967 if (wp->y_tiled) {
1968 int extra_lines;
1969
1970 if (lines % wp->y_min_scanlines == 0)
1971 extra_lines = wp->y_min_scanlines;
1972 else
1973 extra_lines = wp->y_min_scanlines * 2 -
1974 lines % wp->y_min_scanlines;
1975
1976 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1977 wp->plane_blocks_per_line);
1978 } else {
1979 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1980 }
1981 }
1982
1983 if (!skl_wm_has_lines(i915, level))
1984 lines = 0;
1985
1986 if (lines > skl_wm_max_lines(i915)) {
1987 /* reject it */
1988 result->min_ddb_alloc = U16_MAX;
1989 return;
1990 }
1991
1992 /*
1993 * If lines is valid, assume we can use this watermark level
1994 * for now. We'll come back and disable it after we calculate the
1995 * DDB allocation if it turns out we don't actually have enough
1996 * blocks to satisfy it.
1997 */
1998 result->blocks = blocks;
1999 result->lines = lines;
2000 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
2001 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
2002 result->enable = true;
2003
2004 if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
2005 result->can_sagv = latency >= i915->display.sagv.block_time_us;
2006}
2007
2008static void
2009skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
2010 struct intel_plane *plane,
2011 const struct skl_wm_params *wm_params,
2012 struct skl_wm_level *levels)
2013{
2014 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2015 struct skl_wm_level *result_prev = &levels[0];
2016 int level;
2017
2018 for (level = 0; level < i915->display.wm.num_levels; level++) {
2019 struct skl_wm_level *result = &levels[level];
2020 unsigned int latency = skl_wm_latency(i915, level, wm_params);
2021
2022 skl_compute_plane_wm(crtc_state, plane, level, latency,
2023 wm_params, result_prev, result);
2024
2025 result_prev = result;
2026 }
2027}
2028
2029static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2030 struct intel_plane *plane,
2031 const struct skl_wm_params *wm_params,
2032 struct skl_plane_wm *plane_wm)
2033{
2034 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2035 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2036 struct skl_wm_level *levels = plane_wm->wm;
2037 unsigned int latency = 0;
2038
2039 if (i915->display.sagv.block_time_us)
2040 latency = i915->display.sagv.block_time_us +
2041 skl_wm_latency(i915, 0, wm_params);
2042
2043 skl_compute_plane_wm(crtc_state, plane, 0, latency,
2044 wm_params, &levels[0],
2045 sagv_wm);
2046}
2047
2048static void skl_compute_transition_wm(struct drm_i915_private *i915,
2049 struct skl_wm_level *trans_wm,
2050 const struct skl_wm_level *wm0,
2051 const struct skl_wm_params *wp)
2052{
2053 u16 trans_min, trans_amount, trans_y_tile_min;
2054 u16 wm0_blocks, trans_offset, blocks;
2055
2056 /* Transition WM don't make any sense if ipc is disabled */
2057 if (!skl_watermark_ipc_enabled(i915))
2058 return;
2059
2060 /*
2061 * WaDisableTWM:skl,kbl,cfl,bxt
2062 * Transition WM are not recommended by HW team for GEN9
2063 */
2064 if (DISPLAY_VER(i915) == 9)
2065 return;
2066
2067 if (DISPLAY_VER(i915) >= 11)
2068 trans_min = 4;
2069 else
2070 trans_min = 14;
2071
2072 /* Display WA #1140: glk,cnl */
2073 if (DISPLAY_VER(i915) == 10)
2074 trans_amount = 0;
2075 else
2076 trans_amount = 10; /* This is configurable amount */
2077
2078 trans_offset = trans_min + trans_amount;
2079
2080 /*
2081 * The spec asks for Selected Result Blocks for wm0 (the real value),
2082 * not Result Blocks (the integer value). Pay attention to the capital
2083 * letters. The value wm_l0->blocks is actually Result Blocks, but
2084 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2085 * and since we later will have to get the ceiling of the sum in the
2086 * transition watermarks calculation, we can just pretend Selected
2087 * Result Blocks is Result Blocks minus 1 and it should work for the
2088 * current platforms.
2089 */
2090 wm0_blocks = wm0->blocks - 1;
2091
2092 if (wp->y_tiled) {
2093 trans_y_tile_min =
2094 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2095 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2096 } else {
2097 blocks = wm0_blocks + trans_offset;
2098 }
2099 blocks++;
2100
2101 /*
2102 * Just assume we can enable the transition watermark. After
2103 * computing the DDB we'll come back and disable it if that
2104 * assumption turns out to be false.
2105 */
2106 trans_wm->blocks = blocks;
2107 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2108 trans_wm->enable = true;
2109}
2110
2111static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2112 const struct intel_plane_state *plane_state,
2113 struct intel_plane *plane, int color_plane)
2114{
2115 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2116 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2117 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2118 struct skl_wm_params wm_params;
2119 int ret;
2120
2121 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2122 &wm_params, color_plane);
2123 if (ret)
2124 return ret;
2125
2126 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2127
2128 skl_compute_transition_wm(i915, &wm->trans_wm,
2129 &wm->wm[0], &wm_params);
2130
2131 if (DISPLAY_VER(i915) >= 12) {
2132 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2133
2134 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2135 &wm->sagv.wm0, &wm_params);
2136 }
2137
2138 return 0;
2139}
2140
2141static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2142 const struct intel_plane_state *plane_state,
2143 struct intel_plane *plane)
2144{
2145 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2146 struct skl_wm_params wm_params;
2147 int ret;
2148
2149 wm->is_planar = true;
2150
2151 /* uv plane watermarks must also be validated for NV12/Planar */
2152 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2153 &wm_params, 1);
2154 if (ret)
2155 return ret;
2156
2157 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2158
2159 return 0;
2160}
2161
2162static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2163 const struct intel_plane_state *plane_state)
2164{
2165 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2166 enum plane_id plane_id = plane->id;
2167 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2168 const struct drm_framebuffer *fb = plane_state->hw.fb;
2169 int ret;
2170
2171 memset(wm, 0, sizeof(*wm));
2172
2173 if (!intel_wm_plane_visible(crtc_state, plane_state))
2174 return 0;
2175
2176 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2177 plane, 0);
2178 if (ret)
2179 return ret;
2180
2181 if (fb->format->is_yuv && fb->format->num_planes > 1) {
2182 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2183 plane);
2184 if (ret)
2185 return ret;
2186 }
2187
2188 return 0;
2189}
2190
2191static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2192 const struct intel_plane_state *plane_state)
2193{
2194 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2195 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2196 enum plane_id plane_id = plane->id;
2197 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2198 int ret;
2199
2200 /* Watermarks calculated in master */
2201 if (plane_state->planar_slave)
2202 return 0;
2203
2204 memset(wm, 0, sizeof(*wm));
2205
2206 if (plane_state->planar_linked_plane) {
2207 const struct drm_framebuffer *fb = plane_state->hw.fb;
2208
2209 drm_WARN_ON(&i915->drm,
2210 !intel_wm_plane_visible(crtc_state, plane_state));
2211 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2212 fb->format->num_planes == 1);
2213
2214 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2215 plane_state->planar_linked_plane, 0);
2216 if (ret)
2217 return ret;
2218
2219 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2220 plane, 1);
2221 if (ret)
2222 return ret;
2223 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2224 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2225 plane, 0);
2226 if (ret)
2227 return ret;
2228 }
2229
2230 return 0;
2231}
2232
2233static bool
2234skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2235 int wm0_lines, int latency)
2236{
2237 const struct drm_display_mode *adjusted_mode =
2238 &crtc_state->hw.adjusted_mode;
2239
2240 /* FIXME missing scaler and DSC pre-fill time */
2241 return crtc_state->framestart_delay +
2242 intel_usecs_to_scanlines(adjusted_mode, latency) +
2243 wm0_lines >
2244 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2245}
2246
2247static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2248{
2249 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2250 enum plane_id plane_id;
2251 int wm0_lines = 0;
2252
2253 for_each_plane_id_on_crtc(crtc, plane_id) {
2254 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2255
2256 /* FIXME what about !skl_wm_has_lines() platforms? */
2257 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2258 }
2259
2260 return wm0_lines;
2261}
2262
2263static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2264 int wm0_lines)
2265{
2266 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2267 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2268 int level;
2269
2270 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2271 int latency;
2272
2273 /* FIXME should we care about the latency w/a's? */
2274 latency = skl_wm_latency(i915, level, NULL);
2275 if (latency == 0)
2276 continue;
2277
2278 /* FIXME is it correct to use 0 latency for wm0 here? */
2279 if (level == 0)
2280 latency = 0;
2281
2282 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2283 return level;
2284 }
2285
2286 return -EINVAL;
2287}
2288
2289static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2290{
2291 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2292 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2293 int wm0_lines, level;
2294
2295 if (!crtc_state->hw.active)
2296 return 0;
2297
2298 wm0_lines = skl_max_wm0_lines(crtc_state);
2299
2300 level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2301 if (level < 0)
2302 return level;
2303
2304 /*
2305 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2306 * based on whether we're limited by the vblank duration.
2307 */
2308 crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2309
2310 for (level++; level < i915->display.wm.num_levels; level++) {
2311 enum plane_id plane_id;
2312
2313 for_each_plane_id_on_crtc(crtc, plane_id) {
2314 struct skl_plane_wm *wm =
2315 &crtc_state->wm.skl.optimal.planes[plane_id];
2316
2317 /*
2318 * FIXME just clear enable or flag the entire
2319 * thing as bad via min_ddb_alloc=U16_MAX?
2320 */
2321 wm->wm[level].enable = false;
2322 wm->uv_wm[level].enable = false;
2323 }
2324 }
2325
2326 if (DISPLAY_VER(i915) >= 12 &&
2327 i915->display.sagv.block_time_us &&
2328 skl_is_vblank_too_short(crtc_state, wm0_lines,
2329 i915->display.sagv.block_time_us)) {
2330 enum plane_id plane_id;
2331
2332 for_each_plane_id_on_crtc(crtc, plane_id) {
2333 struct skl_plane_wm *wm =
2334 &crtc_state->wm.skl.optimal.planes[plane_id];
2335
2336 wm->sagv.wm0.enable = false;
2337 wm->sagv.trans_wm.enable = false;
2338 }
2339 }
2340
2341 return 0;
2342}
2343
2344static int skl_build_pipe_wm(struct intel_atomic_state *state,
2345 struct intel_crtc *crtc)
2346{
2347 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2348 struct intel_crtc_state *crtc_state =
2349 intel_atomic_get_new_crtc_state(state, crtc);
2350 const struct intel_plane_state *plane_state;
2351 struct intel_plane *plane;
2352 int ret, i;
2353
2354 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2355 /*
2356 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2357 * instead but we don't populate that correctly for NV12 Y
2358 * planes so for now hack this.
2359 */
2360 if (plane->pipe != crtc->pipe)
2361 continue;
2362
2363 if (DISPLAY_VER(i915) >= 11)
2364 ret = icl_build_plane_wm(crtc_state, plane_state);
2365 else
2366 ret = skl_build_plane_wm(crtc_state, plane_state);
2367 if (ret)
2368 return ret;
2369 }
2370
2371 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2372
2373 return skl_wm_check_vblank(crtc_state);
2374}
2375
2376static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2377 const struct skl_wm_level *l2)
2378{
2379 return l1->enable == l2->enable &&
2380 l1->ignore_lines == l2->ignore_lines &&
2381 l1->lines == l2->lines &&
2382 l1->blocks == l2->blocks;
2383}
2384
2385static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2386 const struct skl_plane_wm *wm1,
2387 const struct skl_plane_wm *wm2)
2388{
2389 int level;
2390
2391 for (level = 0; level < i915->display.wm.num_levels; level++) {
2392 /*
2393 * We don't check uv_wm as the hardware doesn't actually
2394 * use it. It only gets used for calculating the required
2395 * ddb allocation.
2396 */
2397 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2398 return false;
2399 }
2400
2401 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2402 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2403 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2404}
2405
2406static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2407 const struct skl_ddb_entry *b)
2408{
2409 return a->start < b->end && b->start < a->end;
2410}
2411
2412static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2413 const struct skl_ddb_entry *b)
2414{
2415 if (a->end && b->end) {
2416 a->start = min(a->start, b->start);
2417 a->end = max(a->end, b->end);
2418 } else if (b->end) {
2419 a->start = b->start;
2420 a->end = b->end;
2421 }
2422}
2423
2424bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2425 const struct skl_ddb_entry *entries,
2426 int num_entries, int ignore_idx)
2427{
2428 int i;
2429
2430 for (i = 0; i < num_entries; i++) {
2431 if (i != ignore_idx &&
2432 skl_ddb_entries_overlap(ddb, &entries[i]))
2433 return true;
2434 }
2435
2436 return false;
2437}
2438
2439static int
2440skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2441 struct intel_crtc *crtc)
2442{
2443 struct drm_i915_private *i915 = to_i915(state->base.dev);
2444 const struct intel_crtc_state *old_crtc_state =
2445 intel_atomic_get_old_crtc_state(state, crtc);
2446 struct intel_crtc_state *new_crtc_state =
2447 intel_atomic_get_new_crtc_state(state, crtc);
2448 struct intel_plane *plane;
2449
2450 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2451 struct intel_plane_state *plane_state;
2452 enum plane_id plane_id = plane->id;
2453
2454 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2455 &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2456 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2457 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2458 continue;
2459
2460 if (new_crtc_state->do_async_flip) {
2461 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2462 plane->base.base.id, plane->base.name);
2463 return -EINVAL;
2464 }
2465
2466 plane_state = intel_atomic_get_plane_state(state, plane);
2467 if (IS_ERR(plane_state))
2468 return PTR_ERR(plane_state);
2469
2470 new_crtc_state->update_planes |= BIT(plane_id);
2471 new_crtc_state->async_flip_planes = 0;
2472 new_crtc_state->do_async_flip = false;
2473 }
2474
2475 return 0;
2476}
2477
2478static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2479{
2480 struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2481 u8 enabled_slices;
2482 enum pipe pipe;
2483
2484 /*
2485 * FIXME: For now we always enable slice S1 as per
2486 * the Bspec display initialization sequence.
2487 */
2488 enabled_slices = BIT(DBUF_S1);
2489
2490 for_each_pipe(i915, pipe)
2491 enabled_slices |= dbuf_state->slices[pipe];
2492
2493 return enabled_slices;
2494}
2495
2496static int
2497skl_compute_ddb(struct intel_atomic_state *state)
2498{
2499 struct drm_i915_private *i915 = to_i915(state->base.dev);
2500 const struct intel_dbuf_state *old_dbuf_state;
2501 struct intel_dbuf_state *new_dbuf_state = NULL;
2502 struct intel_crtc_state *new_crtc_state;
2503 struct intel_crtc *crtc;
2504 int ret, i;
2505
2506 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2507 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2508 if (IS_ERR(new_dbuf_state))
2509 return PTR_ERR(new_dbuf_state);
2510
2511 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2512 break;
2513 }
2514
2515 if (!new_dbuf_state)
2516 return 0;
2517
2518 new_dbuf_state->active_pipes =
2519 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2520
2521 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2522 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2523 if (ret)
2524 return ret;
2525 }
2526
2527 if (HAS_MBUS_JOINING(i915)) {
2528 new_dbuf_state->joined_mbus =
2529 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2530
2531 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2532 ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus);
2533 if (ret)
2534 return ret;
2535 }
2536 }
2537
2538 for_each_intel_crtc(&i915->drm, crtc) {
2539 enum pipe pipe = crtc->pipe;
2540
2541 new_dbuf_state->slices[pipe] =
2542 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2543 new_dbuf_state->joined_mbus);
2544
2545 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2546 continue;
2547
2548 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2549 if (ret)
2550 return ret;
2551 }
2552
2553 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2554
2555 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2556 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2557 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2558 if (ret)
2559 return ret;
2560
2561 drm_dbg_kms(&i915->drm,
2562 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2563 old_dbuf_state->enabled_slices,
2564 new_dbuf_state->enabled_slices,
2565 DISPLAY_INFO(i915)->dbuf.slice_mask,
2566 str_yes_no(old_dbuf_state->joined_mbus),
2567 str_yes_no(new_dbuf_state->joined_mbus));
2568 }
2569
2570 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2571 enum pipe pipe = crtc->pipe;
2572
2573 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2574
2575 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2576 continue;
2577
2578 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2579 if (ret)
2580 return ret;
2581 }
2582
2583 for_each_intel_crtc(&i915->drm, crtc) {
2584 ret = skl_crtc_allocate_ddb(state, crtc);
2585 if (ret)
2586 return ret;
2587 }
2588
2589 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2590 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2591 if (ret)
2592 return ret;
2593
2594 ret = skl_ddb_add_affected_planes(state, crtc);
2595 if (ret)
2596 return ret;
2597 }
2598
2599 return 0;
2600}
2601
2602static char enast(bool enable)
2603{
2604 return enable ? '*' : ' ';
2605}
2606
2607static void
2608skl_print_wm_changes(struct intel_atomic_state *state)
2609{
2610 struct drm_i915_private *i915 = to_i915(state->base.dev);
2611 const struct intel_crtc_state *old_crtc_state;
2612 const struct intel_crtc_state *new_crtc_state;
2613 struct intel_plane *plane;
2614 struct intel_crtc *crtc;
2615 int i;
2616
2617 if (!drm_debug_enabled(DRM_UT_KMS))
2618 return;
2619
2620 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2621 new_crtc_state, i) {
2622 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2623
2624 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2625 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2626
2627 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2628 enum plane_id plane_id = plane->id;
2629 const struct skl_ddb_entry *old, *new;
2630
2631 old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2632 new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2633
2634 if (skl_ddb_entry_equal(old, new))
2635 continue;
2636
2637 drm_dbg_kms(&i915->drm,
2638 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2639 plane->base.base.id, plane->base.name,
2640 old->start, old->end, new->start, new->end,
2641 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2642 }
2643
2644 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2645 enum plane_id plane_id = plane->id;
2646 const struct skl_plane_wm *old_wm, *new_wm;
2647
2648 old_wm = &old_pipe_wm->planes[plane_id];
2649 new_wm = &new_pipe_wm->planes[plane_id];
2650
2651 if (skl_plane_wm_equals(i915, old_wm, new_wm))
2652 continue;
2653
2654 drm_dbg_kms(&i915->drm,
2655 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2656 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2657 plane->base.base.id, plane->base.name,
2658 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2659 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2660 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2661 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2662 enast(old_wm->trans_wm.enable),
2663 enast(old_wm->sagv.wm0.enable),
2664 enast(old_wm->sagv.trans_wm.enable),
2665 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2666 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2667 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2668 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2669 enast(new_wm->trans_wm.enable),
2670 enast(new_wm->sagv.wm0.enable),
2671 enast(new_wm->sagv.trans_wm.enable));
2672
2673 drm_dbg_kms(&i915->drm,
2674 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2675 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2676 plane->base.base.id, plane->base.name,
2677 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2678 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2679 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2680 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2681 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2682 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2683 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2684 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2685 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2686 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2687 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2688 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2689 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2690 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2691 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2692 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2693 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2694 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2695 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2696 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2697 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2698 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2699
2700 drm_dbg_kms(&i915->drm,
2701 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2702 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2703 plane->base.base.id, plane->base.name,
2704 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2705 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2706 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2707 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2708 old_wm->trans_wm.blocks,
2709 old_wm->sagv.wm0.blocks,
2710 old_wm->sagv.trans_wm.blocks,
2711 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2712 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2713 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2714 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2715 new_wm->trans_wm.blocks,
2716 new_wm->sagv.wm0.blocks,
2717 new_wm->sagv.trans_wm.blocks);
2718
2719 drm_dbg_kms(&i915->drm,
2720 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2721 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2722 plane->base.base.id, plane->base.name,
2723 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2724 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2725 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2726 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2727 old_wm->trans_wm.min_ddb_alloc,
2728 old_wm->sagv.wm0.min_ddb_alloc,
2729 old_wm->sagv.trans_wm.min_ddb_alloc,
2730 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2731 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2732 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2733 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2734 new_wm->trans_wm.min_ddb_alloc,
2735 new_wm->sagv.wm0.min_ddb_alloc,
2736 new_wm->sagv.trans_wm.min_ddb_alloc);
2737 }
2738 }
2739}
2740
2741static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2742 const struct skl_pipe_wm *old_pipe_wm,
2743 const struct skl_pipe_wm *new_pipe_wm)
2744{
2745 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2746 int level;
2747
2748 for (level = 0; level < i915->display.wm.num_levels; level++) {
2749 /*
2750 * We don't check uv_wm as the hardware doesn't actually
2751 * use it. It only gets used for calculating the required
2752 * ddb allocation.
2753 */
2754 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2755 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2756 return false;
2757 }
2758
2759 if (HAS_HW_SAGV_WM(i915)) {
2760 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2761 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2762
2763 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2764 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2765 return false;
2766 }
2767
2768 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2769 skl_plane_trans_wm(new_pipe_wm, plane->id));
2770}
2771
2772/*
2773 * To make sure the cursor watermark registers are always consistent
2774 * with our computed state the following scenario needs special
2775 * treatment:
2776 *
2777 * 1. enable cursor
2778 * 2. move cursor entirely offscreen
2779 * 3. disable cursor
2780 *
2781 * Step 2. does call .disable_plane() but does not zero the watermarks
2782 * (since we consider an offscreen cursor still active for the purposes
2783 * of watermarks). Step 3. would not normally call .disable_plane()
2784 * because the actual plane visibility isn't changing, and we don't
2785 * deallocate the cursor ddb until the pipe gets disabled. So we must
2786 * force step 3. to call .disable_plane() to update the watermark
2787 * registers properly.
2788 *
2789 * Other planes do not suffer from this issues as their watermarks are
2790 * calculated based on the actual plane visibility. The only time this
2791 * can trigger for the other planes is during the initial readout as the
2792 * default value of the watermarks registers is not zero.
2793 */
2794static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2795 struct intel_crtc *crtc)
2796{
2797 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2798 const struct intel_crtc_state *old_crtc_state =
2799 intel_atomic_get_old_crtc_state(state, crtc);
2800 struct intel_crtc_state *new_crtc_state =
2801 intel_atomic_get_new_crtc_state(state, crtc);
2802 struct intel_plane *plane;
2803
2804 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2805 struct intel_plane_state *plane_state;
2806 enum plane_id plane_id = plane->id;
2807
2808 /*
2809 * Force a full wm update for every plane on modeset.
2810 * Required because the reset value of the wm registers
2811 * is non-zero, whereas we want all disabled planes to
2812 * have zero watermarks. So if we turn off the relevant
2813 * power well the hardware state will go out of sync
2814 * with the software state.
2815 */
2816 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2817 skl_plane_selected_wm_equals(plane,
2818 &old_crtc_state->wm.skl.optimal,
2819 &new_crtc_state->wm.skl.optimal))
2820 continue;
2821
2822 if (new_crtc_state->do_async_flip) {
2823 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2824 plane->base.base.id, plane->base.name);
2825 return -EINVAL;
2826 }
2827
2828 plane_state = intel_atomic_get_plane_state(state, plane);
2829 if (IS_ERR(plane_state))
2830 return PTR_ERR(plane_state);
2831
2832 new_crtc_state->update_planes |= BIT(plane_id);
2833 new_crtc_state->async_flip_planes = 0;
2834 new_crtc_state->do_async_flip = false;
2835 }
2836
2837 return 0;
2838}
2839
2840/*
2841 * If Fixed Refresh Rate or For VRR case Vmin = Vmax = Flipline:
2842 * Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from
2843 * watermark level1 and up and above. If watermark level 1 is
2844 * invalid program it with all 1's.
2845 * Program PKG_C_LATENCY Added Wake Time = DSB execution time
2846 * If Variable Refresh Rate where Vmin != Vmax != Flipline:
2847 * Program DEEP PKG_C_LATENCY Pkg C with all 1's.
2848 * Program PKG_C_LATENCY Added Wake Time = 0
2849 */
2850static void
2851skl_program_dpkgc_latency(struct drm_i915_private *i915, bool enable_dpkgc)
2852{
2853 u32 max_latency = 0;
2854 u32 clear = 0, val = 0;
2855 u32 added_wake_time = 0;
2856
2857 if (DISPLAY_VER(i915) < 20)
2858 return;
2859
2860 if (enable_dpkgc) {
2861 max_latency = skl_watermark_max_latency(i915, 1);
2862 if (max_latency == 0)
2863 max_latency = LNL_PKG_C_LATENCY_MASK;
2864 added_wake_time = DSB_EXE_TIME +
2865 i915->display.sagv.block_time_us;
2866 } else {
2867 max_latency = LNL_PKG_C_LATENCY_MASK;
2868 added_wake_time = 0;
2869 }
2870
2871 clear |= LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK;
2872 val |= REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, max_latency);
2873 val |= REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time);
2874
2875 intel_uncore_rmw(&i915->uncore, LNL_PKG_C_LATENCY, clear, val);
2876}
2877
2878static int
2879skl_compute_wm(struct intel_atomic_state *state)
2880{
2881 struct intel_crtc *crtc;
2882 struct intel_crtc_state __maybe_unused *new_crtc_state;
2883 int ret, i;
2884 bool enable_dpkgc = false;
2885
2886 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2887 ret = skl_build_pipe_wm(state, crtc);
2888 if (ret)
2889 return ret;
2890 }
2891
2892 ret = skl_compute_ddb(state);
2893 if (ret)
2894 return ret;
2895
2896 ret = intel_compute_sagv_mask(state);
2897 if (ret)
2898 return ret;
2899
2900 /*
2901 * skl_compute_ddb() will have adjusted the final watermarks
2902 * based on how much ddb is available. Now we can actually
2903 * check if the final watermarks changed.
2904 */
2905 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2906 ret = skl_wm_add_affected_planes(state, crtc);
2907 if (ret)
2908 return ret;
2909
2910 if ((new_crtc_state->vrr.vmin == new_crtc_state->vrr.vmax &&
2911 new_crtc_state->vrr.vmin == new_crtc_state->vrr.flipline) ||
2912 !new_crtc_state->vrr.enable)
2913 enable_dpkgc = true;
2914 }
2915
2916 skl_program_dpkgc_latency(to_i915(state->base.dev), enable_dpkgc);
2917
2918 skl_print_wm_changes(state);
2919
2920 return 0;
2921}
2922
2923static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2924{
2925 level->enable = val & PLANE_WM_EN;
2926 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2927 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2928 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2929}
2930
2931static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2932 struct skl_pipe_wm *out)
2933{
2934 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2935 enum pipe pipe = crtc->pipe;
2936 enum plane_id plane_id;
2937 int level;
2938 u32 val;
2939
2940 for_each_plane_id_on_crtc(crtc, plane_id) {
2941 struct skl_plane_wm *wm = &out->planes[plane_id];
2942
2943 for (level = 0; level < i915->display.wm.num_levels; level++) {
2944 if (plane_id != PLANE_CURSOR)
2945 val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2946 else
2947 val = intel_de_read(i915, CUR_WM(pipe, level));
2948
2949 skl_wm_level_from_reg_val(val, &wm->wm[level]);
2950 }
2951
2952 if (plane_id != PLANE_CURSOR)
2953 val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2954 else
2955 val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2956
2957 skl_wm_level_from_reg_val(val, &wm->trans_wm);
2958
2959 if (HAS_HW_SAGV_WM(i915)) {
2960 if (plane_id != PLANE_CURSOR)
2961 val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2962 else
2963 val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2964
2965 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2966
2967 if (plane_id != PLANE_CURSOR)
2968 val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2969 else
2970 val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2971
2972 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2973 } else if (DISPLAY_VER(i915) >= 12) {
2974 wm->sagv.wm0 = wm->wm[0];
2975 wm->sagv.trans_wm = wm->trans_wm;
2976 }
2977 }
2978}
2979
2980static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2981{
2982 struct intel_display *display = &i915->display;
2983 struct intel_dbuf_state *dbuf_state =
2984 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2985 struct intel_crtc *crtc;
2986
2987 if (HAS_MBUS_JOINING(i915))
2988 dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
2989
2990 dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, &display->cdclk.hw);
2991
2992 for_each_intel_crtc(&i915->drm, crtc) {
2993 struct intel_crtc_state *crtc_state =
2994 to_intel_crtc_state(crtc->base.state);
2995 enum pipe pipe = crtc->pipe;
2996 unsigned int mbus_offset;
2997 enum plane_id plane_id;
2998 u8 slices;
2999
3000 memset(&crtc_state->wm.skl.optimal, 0,
3001 sizeof(crtc_state->wm.skl.optimal));
3002 if (crtc_state->hw.active)
3003 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3004 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3005
3006 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3007
3008 for_each_plane_id_on_crtc(crtc, plane_id) {
3009 struct skl_ddb_entry *ddb =
3010 &crtc_state->wm.skl.plane_ddb[plane_id];
3011 struct skl_ddb_entry *ddb_y =
3012 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3013
3014 if (!crtc_state->hw.active)
3015 continue;
3016
3017 skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3018 plane_id, ddb, ddb_y);
3019
3020 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3021 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3022 }
3023
3024 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3025
3026 /*
3027 * Used for checking overlaps, so we need absolute
3028 * offsets instead of MBUS relative offsets.
3029 */
3030 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3031 dbuf_state->joined_mbus);
3032 mbus_offset = mbus_ddb_offset(i915, slices);
3033 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3034 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3035
3036 /* The slices actually used by the planes on the pipe */
3037 dbuf_state->slices[pipe] =
3038 skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3039
3040 drm_dbg_kms(&i915->drm,
3041 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3042 crtc->base.base.id, crtc->base.name,
3043 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3044 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3045 str_yes_no(dbuf_state->joined_mbus));
3046 }
3047
3048 dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3049}
3050
3051static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3052{
3053 const struct intel_dbuf_state *dbuf_state =
3054 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3055 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3056 struct intel_crtc *crtc;
3057
3058 for_each_intel_crtc(&i915->drm, crtc) {
3059 const struct intel_crtc_state *crtc_state =
3060 to_intel_crtc_state(crtc->base.state);
3061
3062 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3063 }
3064
3065 for_each_intel_crtc(&i915->drm, crtc) {
3066 const struct intel_crtc_state *crtc_state =
3067 to_intel_crtc_state(crtc->base.state);
3068 u8 slices;
3069
3070 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3071 dbuf_state->joined_mbus);
3072 if (dbuf_state->slices[crtc->pipe] & ~slices)
3073 return true;
3074
3075 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3076 I915_MAX_PIPES, crtc->pipe))
3077 return true;
3078 }
3079
3080 return false;
3081}
3082
3083static void skl_wm_sanitize(struct drm_i915_private *i915)
3084{
3085 struct intel_crtc *crtc;
3086
3087 /*
3088 * On TGL/RKL (at least) the BIOS likes to assign the planes
3089 * to the wrong DBUF slices. This will cause an infinite loop
3090 * in skl_commit_modeset_enables() as it can't find a way to
3091 * transition between the old bogus DBUF layout to the new
3092 * proper DBUF layout without DBUF allocation overlaps between
3093 * the planes (which cannot be allowed or else the hardware
3094 * may hang). If we detect a bogus DBUF layout just turn off
3095 * all the planes so that skl_commit_modeset_enables() can
3096 * simply ignore them.
3097 */
3098 if (!skl_dbuf_is_misconfigured(i915))
3099 return;
3100
3101 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3102
3103 for_each_intel_crtc(&i915->drm, crtc) {
3104 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3105 const struct intel_plane_state *plane_state =
3106 to_intel_plane_state(plane->base.state);
3107 struct intel_crtc_state *crtc_state =
3108 to_intel_crtc_state(crtc->base.state);
3109
3110 if (plane_state->uapi.visible)
3111 intel_plane_disable_noatomic(crtc, plane);
3112
3113 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3114
3115 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3116 }
3117}
3118
3119static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3120{
3121 skl_wm_get_hw_state(i915);
3122 skl_wm_sanitize(i915);
3123}
3124
3125void intel_wm_state_verify(struct intel_atomic_state *state,
3126 struct intel_crtc *crtc)
3127{
3128 struct drm_i915_private *i915 = to_i915(state->base.dev);
3129 const struct intel_crtc_state *new_crtc_state =
3130 intel_atomic_get_new_crtc_state(state, crtc);
3131 struct skl_hw_state {
3132 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3133 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3134 struct skl_pipe_wm wm;
3135 } *hw;
3136 const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3137 struct intel_plane *plane;
3138 u8 hw_enabled_slices;
3139 int level;
3140
3141 if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3142 return;
3143
3144 hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3145 if (!hw)
3146 return;
3147
3148 skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3149
3150 skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3151
3152 hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3153
3154 if (DISPLAY_VER(i915) >= 11 &&
3155 hw_enabled_slices != i915->display.dbuf.enabled_slices)
3156 drm_err(&i915->drm,
3157 "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3158 i915->display.dbuf.enabled_slices,
3159 hw_enabled_slices);
3160
3161 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3162 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3163 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3164
3165 /* Watermarks */
3166 for (level = 0; level < i915->display.wm.num_levels; level++) {
3167 hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3168 sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3169
3170 if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3171 continue;
3172
3173 drm_err(&i915->drm,
3174 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3175 plane->base.base.id, plane->base.name, level,
3176 sw_wm_level->enable,
3177 sw_wm_level->blocks,
3178 sw_wm_level->lines,
3179 hw_wm_level->enable,
3180 hw_wm_level->blocks,
3181 hw_wm_level->lines);
3182 }
3183
3184 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3185 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3186
3187 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3188 drm_err(&i915->drm,
3189 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3190 plane->base.base.id, plane->base.name,
3191 sw_wm_level->enable,
3192 sw_wm_level->blocks,
3193 sw_wm_level->lines,
3194 hw_wm_level->enable,
3195 hw_wm_level->blocks,
3196 hw_wm_level->lines);
3197 }
3198
3199 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3200 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3201
3202 if (HAS_HW_SAGV_WM(i915) &&
3203 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3204 drm_err(&i915->drm,
3205 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3206 plane->base.base.id, plane->base.name,
3207 sw_wm_level->enable,
3208 sw_wm_level->blocks,
3209 sw_wm_level->lines,
3210 hw_wm_level->enable,
3211 hw_wm_level->blocks,
3212 hw_wm_level->lines);
3213 }
3214
3215 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3216 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3217
3218 if (HAS_HW_SAGV_WM(i915) &&
3219 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3220 drm_err(&i915->drm,
3221 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3222 plane->base.base.id, plane->base.name,
3223 sw_wm_level->enable,
3224 sw_wm_level->blocks,
3225 sw_wm_level->lines,
3226 hw_wm_level->enable,
3227 hw_wm_level->blocks,
3228 hw_wm_level->lines);
3229 }
3230
3231 /* DDB */
3232 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3233 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3234
3235 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3236 drm_err(&i915->drm,
3237 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3238 plane->base.base.id, plane->base.name,
3239 sw_ddb_entry->start, sw_ddb_entry->end,
3240 hw_ddb_entry->start, hw_ddb_entry->end);
3241 }
3242 }
3243
3244 kfree(hw);
3245}
3246
3247bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3248{
3249 return i915->display.wm.ipc_enabled;
3250}
3251
3252void skl_watermark_ipc_update(struct drm_i915_private *i915)
3253{
3254 if (!HAS_IPC(i915))
3255 return;
3256
3257 intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3258 skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3259}
3260
3261static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3262{
3263 /* Display WA #0477 WaDisableIPC: skl */
3264 if (IS_SKYLAKE(i915))
3265 return false;
3266
3267 /* Display WA #1141: SKL:all KBL:all CFL */
3268 if (IS_KABYLAKE(i915) ||
3269 IS_COFFEELAKE(i915) ||
3270 IS_COMETLAKE(i915))
3271 return i915->dram_info.symmetric_memory;
3272
3273 return true;
3274}
3275
3276void skl_watermark_ipc_init(struct drm_i915_private *i915)
3277{
3278 if (!HAS_IPC(i915))
3279 return;
3280
3281 i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3282
3283 skl_watermark_ipc_update(i915);
3284}
3285
3286static void
3287adjust_wm_latency(struct drm_i915_private *i915,
3288 u16 wm[], int num_levels, int read_latency)
3289{
3290 bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3291 int i, level;
3292
3293 /*
3294 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3295 * need to be disabled. We make sure to sanitize the values out
3296 * of the punit to satisfy this requirement.
3297 */
3298 for (level = 1; level < num_levels; level++) {
3299 if (wm[level] == 0) {
3300 for (i = level + 1; i < num_levels; i++)
3301 wm[i] = 0;
3302
3303 num_levels = level;
3304 break;
3305 }
3306 }
3307
3308 /*
3309 * WaWmMemoryReadLatency
3310 *
3311 * punit doesn't take into account the read latency so we need
3312 * to add proper adjustement to each valid level we retrieve
3313 * from the punit when level 0 response data is 0us.
3314 */
3315 if (wm[0] == 0) {
3316 for (level = 0; level < num_levels; level++)
3317 wm[level] += read_latency;
3318 }
3319
3320 /*
3321 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3322 * If we could not get dimm info enable this WA to prevent from
3323 * any underrun. If not able to get Dimm info assume 16GB dimm
3324 * to avoid any underrun.
3325 */
3326 if (wm_lv_0_adjust_needed)
3327 wm[0] += 1;
3328}
3329
3330static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3331{
3332 int num_levels = i915->display.wm.num_levels;
3333 u32 val;
3334
3335 val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3336 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3337 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3338
3339 val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3340 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3341 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3342
3343 val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3344 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3345 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3346
3347 adjust_wm_latency(i915, wm, num_levels, 6);
3348}
3349
3350static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3351{
3352 int num_levels = i915->display.wm.num_levels;
3353 int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3354 int mult = IS_DG2(i915) ? 2 : 1;
3355 u32 val;
3356 int ret;
3357
3358 /* read the first set of memory latencies[0:3] */
3359 val = 0; /* data0 to be programmed to 0 for first set */
3360 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3361 if (ret) {
3362 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3363 return;
3364 }
3365
3366 wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3367 wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3368 wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3369 wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3370
3371 /* read the second set of memory latencies[4:7] */
3372 val = 1; /* data0 to be programmed to 1 for second set */
3373 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3374 if (ret) {
3375 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3376 return;
3377 }
3378
3379 wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3380 wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3381 wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3382 wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3383
3384 adjust_wm_latency(i915, wm, num_levels, read_latency);
3385}
3386
3387static void skl_setup_wm_latency(struct drm_i915_private *i915)
3388{
3389 if (HAS_HW_SAGV_WM(i915))
3390 i915->display.wm.num_levels = 6;
3391 else
3392 i915->display.wm.num_levels = 8;
3393
3394 if (DISPLAY_VER(i915) >= 14)
3395 mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3396 else
3397 skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3398
3399 intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3400}
3401
3402static const struct intel_wm_funcs skl_wm_funcs = {
3403 .compute_global_watermarks = skl_compute_wm,
3404 .get_hw_state = skl_wm_get_hw_state_and_sanitize,
3405};
3406
3407void skl_wm_init(struct drm_i915_private *i915)
3408{
3409 intel_sagv_init(i915);
3410
3411 skl_setup_wm_latency(i915);
3412
3413 i915->display.funcs.wm = &skl_wm_funcs;
3414}
3415
3416static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3417{
3418 struct intel_dbuf_state *dbuf_state;
3419
3420 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3421 if (!dbuf_state)
3422 return NULL;
3423
3424 return &dbuf_state->base;
3425}
3426
3427static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3428 struct intel_global_state *state)
3429{
3430 kfree(state);
3431}
3432
3433static const struct intel_global_state_funcs intel_dbuf_funcs = {
3434 .atomic_duplicate_state = intel_dbuf_duplicate_state,
3435 .atomic_destroy_state = intel_dbuf_destroy_state,
3436};
3437
3438struct intel_dbuf_state *
3439intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3440{
3441 struct drm_i915_private *i915 = to_i915(state->base.dev);
3442 struct intel_global_state *dbuf_state;
3443
3444 dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3445 if (IS_ERR(dbuf_state))
3446 return ERR_CAST(dbuf_state);
3447
3448 return to_intel_dbuf_state(dbuf_state);
3449}
3450
3451int intel_dbuf_init(struct drm_i915_private *i915)
3452{
3453 struct intel_dbuf_state *dbuf_state;
3454
3455 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3456 if (!dbuf_state)
3457 return -ENOMEM;
3458
3459 intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3460 &dbuf_state->base, &intel_dbuf_funcs);
3461
3462 return 0;
3463}
3464
3465static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3466{
3467 switch (pipe) {
3468 case PIPE_A:
3469 return !(active_pipes & BIT(PIPE_D));
3470 case PIPE_D:
3471 return !(active_pipes & BIT(PIPE_A));
3472 case PIPE_B:
3473 return !(active_pipes & BIT(PIPE_C));
3474 case PIPE_C:
3475 return !(active_pipes & BIT(PIPE_B));
3476 default: /* to suppress compiler warning */
3477 MISSING_CASE(pipe);
3478 break;
3479 }
3480
3481 return false;
3482}
3483
3484static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3485{
3486 struct drm_i915_private *i915 = to_i915(state->base.dev);
3487 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3488 const struct intel_crtc *crtc;
3489 u32 val = 0;
3490
3491 if (DISPLAY_VER(i915) < 11)
3492 return;
3493
3494 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3495 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3496 if (!new_dbuf_state ||
3497 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3498 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3499 return;
3500
3501 if (DISPLAY_VER(i915) >= 14)
3502 val |= MBUS_DBOX_I_CREDIT(2);
3503
3504 if (DISPLAY_VER(i915) >= 12) {
3505 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3506 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3507 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3508 }
3509
3510 if (DISPLAY_VER(i915) >= 14)
3511 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3512 MBUS_DBOX_A_CREDIT(8);
3513 else if (IS_ALDERLAKE_P(i915))
3514 /* Wa_22010947358:adl-p */
3515 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3516 MBUS_DBOX_A_CREDIT(4);
3517 else
3518 val |= MBUS_DBOX_A_CREDIT(2);
3519
3520 if (DISPLAY_VER(i915) >= 14) {
3521 val |= MBUS_DBOX_B_CREDIT(0xA);
3522 } else if (IS_ALDERLAKE_P(i915)) {
3523 val |= MBUS_DBOX_BW_CREDIT(2);
3524 val |= MBUS_DBOX_B_CREDIT(8);
3525 } else if (DISPLAY_VER(i915) >= 12) {
3526 val |= MBUS_DBOX_BW_CREDIT(2);
3527 val |= MBUS_DBOX_B_CREDIT(12);
3528 } else {
3529 val |= MBUS_DBOX_BW_CREDIT(1);
3530 val |= MBUS_DBOX_B_CREDIT(8);
3531 }
3532
3533 for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, new_dbuf_state->active_pipes) {
3534 u32 pipe_val = val;
3535
3536 if (DISPLAY_VERx100(i915) == 1400) {
3537 if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3538 new_dbuf_state->active_pipes))
3539 pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3540 else
3541 pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3542 }
3543
3544 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3545 }
3546}
3547
3548int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3549 int ratio)
3550{
3551 struct intel_dbuf_state *dbuf_state;
3552
3553 dbuf_state = intel_atomic_get_dbuf_state(state);
3554 if (IS_ERR(dbuf_state))
3555 return PTR_ERR(dbuf_state);
3556
3557 dbuf_state->mdclk_cdclk_ratio = ratio;
3558
3559 return intel_atomic_lock_global_state(&dbuf_state->base);
3560}
3561
3562void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
3563 int ratio, bool joined_mbus)
3564{
3565 enum dbuf_slice slice;
3566
3567 if (!HAS_MBUS_JOINING(i915))
3568 return;
3569
3570 if (DISPLAY_VER(i915) >= 20)
3571 intel_de_rmw(i915, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3572 MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3573
3574 if (joined_mbus)
3575 ratio *= 2;
3576
3577 drm_dbg_kms(&i915->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3578 ratio, str_yes_no(joined_mbus));
3579
3580 for_each_dbuf_slice(i915, slice)
3581 intel_de_rmw(i915, DBUF_CTL_S(slice),
3582 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3583 DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3584}
3585
3586static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3587{
3588 struct drm_i915_private *i915 = to_i915(state->base.dev);
3589 const struct intel_dbuf_state *old_dbuf_state =
3590 intel_atomic_get_old_dbuf_state(state);
3591 const struct intel_dbuf_state *new_dbuf_state =
3592 intel_atomic_get_new_dbuf_state(state);
3593 int mdclk_cdclk_ratio;
3594
3595 if (intel_cdclk_is_decreasing_later(state)) {
3596 /* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3597 mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3598 } else {
3599 /* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3600 mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3601 }
3602
3603 intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio,
3604 new_dbuf_state->joined_mbus);
3605}
3606
3607static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3608 const struct intel_dbuf_state *dbuf_state)
3609{
3610 struct intel_display *display = to_intel_display(state);
3611 struct drm_i915_private *i915 = to_i915(state->base.dev);
3612 enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3613 const struct intel_crtc_state *new_crtc_state;
3614 struct intel_crtc *crtc;
3615
3616 drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus);
3617 drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes));
3618
3619 crtc = intel_crtc_for_pipe(display, pipe);
3620 new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3621
3622 if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state))
3623 return pipe;
3624 else
3625 return INVALID_PIPE;
3626}
3627
3628static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3629 enum pipe pipe)
3630{
3631 struct drm_i915_private *i915 = to_i915(state->base.dev);
3632 const struct intel_dbuf_state *old_dbuf_state =
3633 intel_atomic_get_old_dbuf_state(state);
3634 const struct intel_dbuf_state *new_dbuf_state =
3635 intel_atomic_get_new_dbuf_state(state);
3636 u32 mbus_ctl;
3637
3638 drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3639 str_yes_no(old_dbuf_state->joined_mbus),
3640 str_yes_no(new_dbuf_state->joined_mbus),
3641 pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3642
3643 if (new_dbuf_state->joined_mbus)
3644 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3645 else
3646 mbus_ctl = MBUS_HASHING_MODE_2x2;
3647
3648 if (pipe != INVALID_PIPE)
3649 mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3650 else
3651 mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3652
3653 intel_de_rmw(i915, MBUS_CTL,
3654 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3655 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3656}
3657
3658void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3659{
3660 const struct intel_dbuf_state *new_dbuf_state =
3661 intel_atomic_get_new_dbuf_state(state);
3662 const struct intel_dbuf_state *old_dbuf_state =
3663 intel_atomic_get_old_dbuf_state(state);
3664
3665 if (!new_dbuf_state)
3666 return;
3667
3668 if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3669 enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state);
3670
3671 WARN_ON(!new_dbuf_state->base.changed);
3672
3673 intel_dbuf_mbus_join_update(state, pipe);
3674 intel_mbus_dbox_update(state);
3675 intel_dbuf_mdclk_min_tracker_update(state);
3676 }
3677}
3678
3679void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3680{
3681 struct intel_display *display = to_intel_display(state);
3682 const struct intel_dbuf_state *new_dbuf_state =
3683 intel_atomic_get_new_dbuf_state(state);
3684 const struct intel_dbuf_state *old_dbuf_state =
3685 intel_atomic_get_old_dbuf_state(state);
3686
3687 if (!new_dbuf_state)
3688 return;
3689
3690 if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3691 enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state);
3692
3693 WARN_ON(!new_dbuf_state->base.changed);
3694
3695 intel_dbuf_mdclk_min_tracker_update(state);
3696 intel_mbus_dbox_update(state);
3697 intel_dbuf_mbus_join_update(state, pipe);
3698
3699 if (pipe != INVALID_PIPE) {
3700 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3701
3702 intel_crtc_wait_for_next_vblank(crtc);
3703 }
3704 } else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3705 old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3706 WARN_ON(!new_dbuf_state->base.changed);
3707
3708 intel_dbuf_mdclk_min_tracker_update(state);
3709 intel_mbus_dbox_update(state);
3710 }
3711
3712}
3713
3714void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3715{
3716 struct drm_i915_private *i915 = to_i915(state->base.dev);
3717 const struct intel_dbuf_state *new_dbuf_state =
3718 intel_atomic_get_new_dbuf_state(state);
3719 const struct intel_dbuf_state *old_dbuf_state =
3720 intel_atomic_get_old_dbuf_state(state);
3721 u8 old_slices, new_slices;
3722
3723 if (!new_dbuf_state)
3724 return;
3725
3726 old_slices = old_dbuf_state->enabled_slices;
3727 new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3728
3729 if (old_slices == new_slices)
3730 return;
3731
3732 WARN_ON(!new_dbuf_state->base.changed);
3733
3734 gen9_dbuf_slices_update(i915, new_slices);
3735}
3736
3737void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3738{
3739 struct drm_i915_private *i915 = to_i915(state->base.dev);
3740 const struct intel_dbuf_state *new_dbuf_state =
3741 intel_atomic_get_new_dbuf_state(state);
3742 const struct intel_dbuf_state *old_dbuf_state =
3743 intel_atomic_get_old_dbuf_state(state);
3744 u8 old_slices, new_slices;
3745
3746 if (!new_dbuf_state)
3747 return;
3748
3749 old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3750 new_slices = new_dbuf_state->enabled_slices;
3751
3752 if (old_slices == new_slices)
3753 return;
3754
3755 WARN_ON(!new_dbuf_state->base.changed);
3756
3757 gen9_dbuf_slices_update(i915, new_slices);
3758}
3759
3760static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3761{
3762 struct drm_i915_private *i915 = m->private;
3763
3764 seq_printf(m, "Isochronous Priority Control: %s\n",
3765 str_yes_no(skl_watermark_ipc_enabled(i915)));
3766 return 0;
3767}
3768
3769static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3770{
3771 struct drm_i915_private *i915 = inode->i_private;
3772
3773 return single_open(file, skl_watermark_ipc_status_show, i915);
3774}
3775
3776static ssize_t skl_watermark_ipc_status_write(struct file *file,
3777 const char __user *ubuf,
3778 size_t len, loff_t *offp)
3779{
3780 struct seq_file *m = file->private_data;
3781 struct drm_i915_private *i915 = m->private;
3782 intel_wakeref_t wakeref;
3783 bool enable;
3784 int ret;
3785
3786 ret = kstrtobool_from_user(ubuf, len, &enable);
3787 if (ret < 0)
3788 return ret;
3789
3790 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3791 if (!skl_watermark_ipc_enabled(i915) && enable)
3792 drm_info(&i915->drm,
3793 "Enabling IPC: WM will be proper only after next commit\n");
3794 i915->display.wm.ipc_enabled = enable;
3795 skl_watermark_ipc_update(i915);
3796 }
3797
3798 return len;
3799}
3800
3801static const struct file_operations skl_watermark_ipc_status_fops = {
3802 .owner = THIS_MODULE,
3803 .open = skl_watermark_ipc_status_open,
3804 .read = seq_read,
3805 .llseek = seq_lseek,
3806 .release = single_release,
3807 .write = skl_watermark_ipc_status_write
3808};
3809
3810static int intel_sagv_status_show(struct seq_file *m, void *unused)
3811{
3812 struct drm_i915_private *i915 = m->private;
3813 static const char * const sagv_status[] = {
3814 [I915_SAGV_UNKNOWN] = "unknown",
3815 [I915_SAGV_DISABLED] = "disabled",
3816 [I915_SAGV_ENABLED] = "enabled",
3817 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
3818 };
3819
3820 seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3821 seq_printf(m, "SAGV modparam: %s\n",
3822 str_enabled_disabled(i915->display.params.enable_sagv));
3823 seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3824 seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3825
3826 return 0;
3827}
3828
3829DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3830
3831void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3832{
3833 struct drm_minor *minor = i915->drm.primary;
3834
3835 if (HAS_IPC(i915))
3836 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3837 &skl_watermark_ipc_status_fops);
3838
3839 if (HAS_SAGV(i915))
3840 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3841 &intel_sagv_status_fops);
3842}
3843
3844unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level)
3845{
3846 int level;
3847
3848 for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) {
3849 unsigned int latency = skl_wm_latency(i915, level, NULL);
3850
3851 if (latency)
3852 return latency;
3853 }
3854
3855 return 0;
3856}