Loading...
Note: File does not exist in v5.14.15.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <drm/drm_blend.h>
7
8#include "intel_atomic.h"
9#include "intel_atomic_plane.h"
10#include "intel_bw.h"
11#include "intel_de.h"
12#include "intel_display.h"
13#include "intel_display_power.h"
14#include "intel_display_types.h"
15#include "intel_fb.h"
16#include "skl_watermark.h"
17
18#include "i915_drv.h"
19#include "i915_fixed.h"
20#include "i915_reg.h"
21#include "intel_pcode.h"
22#include "intel_pm.h"
23
24static void skl_sagv_disable(struct drm_i915_private *i915);
25
26/* Stores plane specific WM parameters */
27struct skl_wm_params {
28 bool x_tiled, y_tiled;
29 bool rc_surface;
30 bool is_planar;
31 u32 width;
32 u8 cpp;
33 u32 plane_pixel_rate;
34 u32 y_min_scanlines;
35 u32 plane_bytes_per_line;
36 uint_fixed_16_16_t plane_blocks_per_line;
37 uint_fixed_16_16_t y_tile_minimum;
38 u32 linetime_us;
39 u32 dbuf_block_size;
40};
41
42u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
43{
44 u8 enabled_slices = 0;
45 enum dbuf_slice slice;
46
47 for_each_dbuf_slice(i915, slice) {
48 if (intel_uncore_read(&i915->uncore,
49 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
50 enabled_slices |= BIT(slice);
51 }
52
53 return enabled_slices;
54}
55
56/*
57 * FIXME: We still don't have the proper code detect if we need to apply the WA,
58 * so assume we'll always need it in order to avoid underruns.
59 */
60static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
61{
62 return DISPLAY_VER(i915) == 9;
63}
64
65static bool
66intel_has_sagv(struct drm_i915_private *i915)
67{
68 return DISPLAY_VER(i915) >= 9 && !IS_LP(i915) &&
69 i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
70}
71
72static u32
73intel_sagv_block_time(struct drm_i915_private *i915)
74{
75 if (DISPLAY_VER(i915) >= 14) {
76 u32 val;
77
78 val = intel_uncore_read(&i915->uncore, MTL_LATENCY_SAGV);
79
80 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
81 } else if (DISPLAY_VER(i915) >= 12) {
82 u32 val = 0;
83 int ret;
84
85 ret = snb_pcode_read(&i915->uncore,
86 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
87 &val, NULL);
88 if (ret) {
89 drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
90 return 0;
91 }
92
93 return val;
94 } else if (DISPLAY_VER(i915) == 11) {
95 return 10;
96 } else if (DISPLAY_VER(i915) == 9 && !IS_LP(i915)) {
97 return 30;
98 } else {
99 return 0;
100 }
101}
102
103static void intel_sagv_init(struct drm_i915_private *i915)
104{
105 if (!intel_has_sagv(i915))
106 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
107
108 /*
109 * Probe to see if we have working SAGV control.
110 * For icl+ this was already determined by intel_bw_init_hw().
111 */
112 if (DISPLAY_VER(i915) < 11)
113 skl_sagv_disable(i915);
114
115 drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
116
117 i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
118
119 drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
120 str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
121
122 /* avoid overflow when adding with wm0 latency/etc. */
123 if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
124 "Excessive SAGV block time %u, ignoring\n",
125 i915->display.sagv.block_time_us))
126 i915->display.sagv.block_time_us = 0;
127
128 if (!intel_has_sagv(i915))
129 i915->display.sagv.block_time_us = 0;
130}
131
132/*
133 * SAGV dynamically adjusts the system agent voltage and clock frequencies
134 * depending on power and performance requirements. The display engine access
135 * to system memory is blocked during the adjustment time. Because of the
136 * blocking time, having this enabled can cause full system hangs and/or pipe
137 * underruns if we don't meet all of the following requirements:
138 *
139 * - <= 1 pipe enabled
140 * - All planes can enable watermarks for latencies >= SAGV engine block time
141 * - We're not using an interlaced display configuration
142 */
143static void skl_sagv_enable(struct drm_i915_private *i915)
144{
145 int ret;
146
147 if (!intel_has_sagv(i915))
148 return;
149
150 if (i915->display.sagv.status == I915_SAGV_ENABLED)
151 return;
152
153 drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
154 ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
155 GEN9_SAGV_ENABLE);
156
157 /* We don't need to wait for SAGV when enabling */
158
159 /*
160 * Some skl systems, pre-release machines in particular,
161 * don't actually have SAGV.
162 */
163 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
164 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
165 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
166 return;
167 } else if (ret < 0) {
168 drm_err(&i915->drm, "Failed to enable SAGV\n");
169 return;
170 }
171
172 i915->display.sagv.status = I915_SAGV_ENABLED;
173}
174
175static void skl_sagv_disable(struct drm_i915_private *i915)
176{
177 int ret;
178
179 if (!intel_has_sagv(i915))
180 return;
181
182 if (i915->display.sagv.status == I915_SAGV_DISABLED)
183 return;
184
185 drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
186 /* bspec says to keep retrying for at least 1 ms */
187 ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
188 GEN9_SAGV_DISABLE,
189 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
190 1);
191 /*
192 * Some skl systems, pre-release machines in particular,
193 * don't actually have SAGV.
194 */
195 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
196 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
197 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
198 return;
199 } else if (ret < 0) {
200 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
201 return;
202 }
203
204 i915->display.sagv.status = I915_SAGV_DISABLED;
205}
206
207static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
208{
209 struct drm_i915_private *i915 = to_i915(state->base.dev);
210 const struct intel_bw_state *new_bw_state =
211 intel_atomic_get_new_bw_state(state);
212
213 if (!new_bw_state)
214 return;
215
216 if (!intel_can_enable_sagv(i915, new_bw_state))
217 skl_sagv_disable(i915);
218}
219
220static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
221{
222 struct drm_i915_private *i915 = to_i915(state->base.dev);
223 const struct intel_bw_state *new_bw_state =
224 intel_atomic_get_new_bw_state(state);
225
226 if (!new_bw_state)
227 return;
228
229 if (intel_can_enable_sagv(i915, new_bw_state))
230 skl_sagv_enable(i915);
231}
232
233static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
234{
235 struct drm_i915_private *i915 = to_i915(state->base.dev);
236 const struct intel_bw_state *old_bw_state =
237 intel_atomic_get_old_bw_state(state);
238 const struct intel_bw_state *new_bw_state =
239 intel_atomic_get_new_bw_state(state);
240 u16 old_mask, new_mask;
241
242 if (!new_bw_state)
243 return;
244
245 old_mask = old_bw_state->qgv_points_mask;
246 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
247
248 if (old_mask == new_mask)
249 return;
250
251 WARN_ON(!new_bw_state->base.changed);
252
253 drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
254 old_mask, new_mask);
255
256 /*
257 * Restrict required qgv points before updating the configuration.
258 * According to BSpec we can't mask and unmask qgv points at the same
259 * time. Also masking should be done before updating the configuration
260 * and unmasking afterwards.
261 */
262 icl_pcode_restrict_qgv_points(i915, new_mask);
263}
264
265static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
266{
267 struct drm_i915_private *i915 = to_i915(state->base.dev);
268 const struct intel_bw_state *old_bw_state =
269 intel_atomic_get_old_bw_state(state);
270 const struct intel_bw_state *new_bw_state =
271 intel_atomic_get_new_bw_state(state);
272 u16 old_mask, new_mask;
273
274 if (!new_bw_state)
275 return;
276
277 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
278 new_mask = new_bw_state->qgv_points_mask;
279
280 if (old_mask == new_mask)
281 return;
282
283 WARN_ON(!new_bw_state->base.changed);
284
285 drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
286 old_mask, new_mask);
287
288 /*
289 * Allow required qgv points after updating the configuration.
290 * According to BSpec we can't mask and unmask qgv points at the same
291 * time. Also masking should be done before updating the configuration
292 * and unmasking afterwards.
293 */
294 icl_pcode_restrict_qgv_points(i915, new_mask);
295}
296
297void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
298{
299 struct drm_i915_private *i915 = to_i915(state->base.dev);
300
301 /*
302 * Just return if we can't control SAGV or don't have it.
303 * This is different from situation when we have SAGV but just can't
304 * afford it due to DBuf limitation - in case if SAGV is completely
305 * disabled in a BIOS, we are not even allowed to send a PCode request,
306 * as it will throw an error. So have to check it here.
307 */
308 if (!intel_has_sagv(i915))
309 return;
310
311 if (DISPLAY_VER(i915) >= 11)
312 icl_sagv_pre_plane_update(state);
313 else
314 skl_sagv_pre_plane_update(state);
315}
316
317void intel_sagv_post_plane_update(struct intel_atomic_state *state)
318{
319 struct drm_i915_private *i915 = to_i915(state->base.dev);
320
321 /*
322 * Just return if we can't control SAGV or don't have it.
323 * This is different from situation when we have SAGV but just can't
324 * afford it due to DBuf limitation - in case if SAGV is completely
325 * disabled in a BIOS, we are not even allowed to send a PCode request,
326 * as it will throw an error. So have to check it here.
327 */
328 if (!intel_has_sagv(i915))
329 return;
330
331 if (DISPLAY_VER(i915) >= 11)
332 icl_sagv_post_plane_update(state);
333 else
334 skl_sagv_post_plane_update(state);
335}
336
337static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
338{
339 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
340 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
341 enum plane_id plane_id;
342 int max_level = INT_MAX;
343
344 if (!intel_has_sagv(i915))
345 return false;
346
347 if (!crtc_state->hw.active)
348 return true;
349
350 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
351 return false;
352
353 for_each_plane_id_on_crtc(crtc, plane_id) {
354 const struct skl_plane_wm *wm =
355 &crtc_state->wm.skl.optimal.planes[plane_id];
356 int level;
357
358 /* Skip this plane if it's not enabled */
359 if (!wm->wm[0].enable)
360 continue;
361
362 /* Find the highest enabled wm level for this plane */
363 for (level = ilk_wm_max_level(i915);
364 !wm->wm[level].enable; --level)
365 { }
366
367 /* Highest common enabled wm level for all planes */
368 max_level = min(level, max_level);
369 }
370
371 /* No enabled planes? */
372 if (max_level == INT_MAX)
373 return true;
374
375 for_each_plane_id_on_crtc(crtc, plane_id) {
376 const struct skl_plane_wm *wm =
377 &crtc_state->wm.skl.optimal.planes[plane_id];
378
379 /*
380 * All enabled planes must have enabled a common wm level that
381 * can tolerate memory latencies higher than sagv_block_time_us
382 */
383 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
384 return false;
385 }
386
387 return true;
388}
389
390static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
391{
392 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
393 enum plane_id plane_id;
394
395 if (!crtc_state->hw.active)
396 return true;
397
398 for_each_plane_id_on_crtc(crtc, plane_id) {
399 const struct skl_plane_wm *wm =
400 &crtc_state->wm.skl.optimal.planes[plane_id];
401
402 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
403 return false;
404 }
405
406 return true;
407}
408
409static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
410{
411 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
412 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
413
414 if (DISPLAY_VER(i915) >= 12)
415 return tgl_crtc_can_enable_sagv(crtc_state);
416 else
417 return skl_crtc_can_enable_sagv(crtc_state);
418}
419
420bool intel_can_enable_sagv(struct drm_i915_private *i915,
421 const struct intel_bw_state *bw_state)
422{
423 if (DISPLAY_VER(i915) < 11 &&
424 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
425 return false;
426
427 return bw_state->pipe_sagv_reject == 0;
428}
429
430static int intel_compute_sagv_mask(struct intel_atomic_state *state)
431{
432 struct drm_i915_private *i915 = to_i915(state->base.dev);
433 int ret;
434 struct intel_crtc *crtc;
435 struct intel_crtc_state *new_crtc_state;
436 struct intel_bw_state *new_bw_state = NULL;
437 const struct intel_bw_state *old_bw_state = NULL;
438 int i;
439
440 for_each_new_intel_crtc_in_state(state, crtc,
441 new_crtc_state, i) {
442 new_bw_state = intel_atomic_get_bw_state(state);
443 if (IS_ERR(new_bw_state))
444 return PTR_ERR(new_bw_state);
445
446 old_bw_state = intel_atomic_get_old_bw_state(state);
447
448 if (intel_crtc_can_enable_sagv(new_crtc_state))
449 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
450 else
451 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
452 }
453
454 if (!new_bw_state)
455 return 0;
456
457 new_bw_state->active_pipes =
458 intel_calc_active_pipes(state, old_bw_state->active_pipes);
459
460 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
461 ret = intel_atomic_lock_global_state(&new_bw_state->base);
462 if (ret)
463 return ret;
464 }
465
466 if (intel_can_enable_sagv(i915, new_bw_state) !=
467 intel_can_enable_sagv(i915, old_bw_state)) {
468 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
469 if (ret)
470 return ret;
471 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
472 ret = intel_atomic_lock_global_state(&new_bw_state->base);
473 if (ret)
474 return ret;
475 }
476
477 for_each_new_intel_crtc_in_state(state, crtc,
478 new_crtc_state, i) {
479 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
480
481 /*
482 * We store use_sagv_wm in the crtc state rather than relying on
483 * that bw state since we have no convenient way to get at the
484 * latter from the plane commit hooks (especially in the legacy
485 * cursor case)
486 */
487 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
488 DISPLAY_VER(i915) >= 12 &&
489 intel_can_enable_sagv(i915, new_bw_state);
490 }
491
492 return 0;
493}
494
495static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
496 u16 start, u16 end)
497{
498 entry->start = start;
499 entry->end = end;
500
501 return end;
502}
503
504static int intel_dbuf_slice_size(struct drm_i915_private *i915)
505{
506 return INTEL_INFO(i915)->display.dbuf.size /
507 hweight8(INTEL_INFO(i915)->display.dbuf.slice_mask);
508}
509
510static void
511skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
512 struct skl_ddb_entry *ddb)
513{
514 int slice_size = intel_dbuf_slice_size(i915);
515
516 if (!slice_mask) {
517 ddb->start = 0;
518 ddb->end = 0;
519 return;
520 }
521
522 ddb->start = (ffs(slice_mask) - 1) * slice_size;
523 ddb->end = fls(slice_mask) * slice_size;
524
525 WARN_ON(ddb->start >= ddb->end);
526 WARN_ON(ddb->end > INTEL_INFO(i915)->display.dbuf.size);
527}
528
529static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
530{
531 struct skl_ddb_entry ddb;
532
533 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
534 slice_mask = BIT(DBUF_S1);
535 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
536 slice_mask = BIT(DBUF_S3);
537
538 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
539
540 return ddb.start;
541}
542
543u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
544 const struct skl_ddb_entry *entry)
545{
546 int slice_size = intel_dbuf_slice_size(i915);
547 enum dbuf_slice start_slice, end_slice;
548 u8 slice_mask = 0;
549
550 if (!skl_ddb_entry_size(entry))
551 return 0;
552
553 start_slice = entry->start / slice_size;
554 end_slice = (entry->end - 1) / slice_size;
555
556 /*
557 * Per plane DDB entry can in a really worst case be on multiple slices
558 * but single entry is anyway contigious.
559 */
560 while (start_slice <= end_slice) {
561 slice_mask |= BIT(start_slice);
562 start_slice++;
563 }
564
565 return slice_mask;
566}
567
568static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
569{
570 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
571 int hdisplay, vdisplay;
572
573 if (!crtc_state->hw.active)
574 return 0;
575
576 /*
577 * Watermark/ddb requirement highly depends upon width of the
578 * framebuffer, So instead of allocating DDB equally among pipes
579 * distribute DDB based on resolution/width of the display.
580 */
581 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
582
583 return hdisplay;
584}
585
586static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
587 enum pipe for_pipe,
588 unsigned int *weight_start,
589 unsigned int *weight_end,
590 unsigned int *weight_total)
591{
592 struct drm_i915_private *i915 =
593 to_i915(dbuf_state->base.state->base.dev);
594 enum pipe pipe;
595
596 *weight_start = 0;
597 *weight_end = 0;
598 *weight_total = 0;
599
600 for_each_pipe(i915, pipe) {
601 int weight = dbuf_state->weight[pipe];
602
603 /*
604 * Do not account pipes using other slice sets
605 * luckily as of current BSpec slice sets do not partially
606 * intersect(pipes share either same one slice or same slice set
607 * i.e no partial intersection), so it is enough to check for
608 * equality for now.
609 */
610 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
611 continue;
612
613 *weight_total += weight;
614 if (pipe < for_pipe) {
615 *weight_start += weight;
616 *weight_end += weight;
617 } else if (pipe == for_pipe) {
618 *weight_end += weight;
619 }
620 }
621}
622
623static int
624skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
625{
626 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
627 unsigned int weight_total, weight_start, weight_end;
628 const struct intel_dbuf_state *old_dbuf_state =
629 intel_atomic_get_old_dbuf_state(state);
630 struct intel_dbuf_state *new_dbuf_state =
631 intel_atomic_get_new_dbuf_state(state);
632 struct intel_crtc_state *crtc_state;
633 struct skl_ddb_entry ddb_slices;
634 enum pipe pipe = crtc->pipe;
635 unsigned int mbus_offset = 0;
636 u32 ddb_range_size;
637 u32 dbuf_slice_mask;
638 u32 start, end;
639 int ret;
640
641 if (new_dbuf_state->weight[pipe] == 0) {
642 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
643 goto out;
644 }
645
646 dbuf_slice_mask = new_dbuf_state->slices[pipe];
647
648 skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
649 mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
650 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
651
652 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
653 &weight_start, &weight_end, &weight_total);
654
655 start = ddb_range_size * weight_start / weight_total;
656 end = ddb_range_size * weight_end / weight_total;
657
658 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
659 ddb_slices.start - mbus_offset + start,
660 ddb_slices.start - mbus_offset + end);
661
662out:
663 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
664 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
665 &new_dbuf_state->ddb[pipe]))
666 return 0;
667
668 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
669 if (ret)
670 return ret;
671
672 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
673 if (IS_ERR(crtc_state))
674 return PTR_ERR(crtc_state);
675
676 /*
677 * Used for checking overlaps, so we need absolute
678 * offsets instead of MBUS relative offsets.
679 */
680 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
681 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
682
683 drm_dbg_kms(&i915->drm,
684 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
685 crtc->base.base.id, crtc->base.name,
686 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
687 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
688 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
689 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
690
691 return 0;
692}
693
694static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
695 int width, const struct drm_format_info *format,
696 u64 modifier, unsigned int rotation,
697 u32 plane_pixel_rate, struct skl_wm_params *wp,
698 int color_plane);
699
700static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
701 struct intel_plane *plane,
702 int level,
703 unsigned int latency,
704 const struct skl_wm_params *wp,
705 const struct skl_wm_level *result_prev,
706 struct skl_wm_level *result /* out */);
707
708static unsigned int
709skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
710 int num_active)
711{
712 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
713 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
714 int level, max_level = ilk_wm_max_level(i915);
715 struct skl_wm_level wm = {};
716 int ret, min_ddb_alloc = 0;
717 struct skl_wm_params wp;
718
719 ret = skl_compute_wm_params(crtc_state, 256,
720 drm_format_info(DRM_FORMAT_ARGB8888),
721 DRM_FORMAT_MOD_LINEAR,
722 DRM_MODE_ROTATE_0,
723 crtc_state->pixel_rate, &wp, 0);
724 drm_WARN_ON(&i915->drm, ret);
725
726 for (level = 0; level <= max_level; level++) {
727 unsigned int latency = i915->display.wm.skl_latency[level];
728
729 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
730 if (wm.min_ddb_alloc == U16_MAX)
731 break;
732
733 min_ddb_alloc = wm.min_ddb_alloc;
734 }
735
736 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
737}
738
739static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
740{
741 skl_ddb_entry_init(entry,
742 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
743 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
744 if (entry->end)
745 entry->end++;
746}
747
748static void
749skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
750 const enum pipe pipe,
751 const enum plane_id plane_id,
752 struct skl_ddb_entry *ddb,
753 struct skl_ddb_entry *ddb_y)
754{
755 u32 val;
756
757 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
758 if (plane_id == PLANE_CURSOR) {
759 val = intel_uncore_read(&i915->uncore, CUR_BUF_CFG(pipe));
760 skl_ddb_entry_init_from_hw(ddb, val);
761 return;
762 }
763
764 val = intel_uncore_read(&i915->uncore, PLANE_BUF_CFG(pipe, plane_id));
765 skl_ddb_entry_init_from_hw(ddb, val);
766
767 if (DISPLAY_VER(i915) >= 11)
768 return;
769
770 val = intel_uncore_read(&i915->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
771 skl_ddb_entry_init_from_hw(ddb_y, val);
772}
773
774static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
775 struct skl_ddb_entry *ddb,
776 struct skl_ddb_entry *ddb_y)
777{
778 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
779 enum intel_display_power_domain power_domain;
780 enum pipe pipe = crtc->pipe;
781 intel_wakeref_t wakeref;
782 enum plane_id plane_id;
783
784 power_domain = POWER_DOMAIN_PIPE(pipe);
785 wakeref = intel_display_power_get_if_enabled(i915, power_domain);
786 if (!wakeref)
787 return;
788
789 for_each_plane_id_on_crtc(crtc, plane_id)
790 skl_ddb_get_hw_plane_state(i915, pipe,
791 plane_id,
792 &ddb[plane_id],
793 &ddb_y[plane_id]);
794
795 intel_display_power_put(i915, power_domain, wakeref);
796}
797
798struct dbuf_slice_conf_entry {
799 u8 active_pipes;
800 u8 dbuf_mask[I915_MAX_PIPES];
801 bool join_mbus;
802};
803
804/*
805 * Table taken from Bspec 12716
806 * Pipes do have some preferred DBuf slice affinity,
807 * plus there are some hardcoded requirements on how
808 * those should be distributed for multipipe scenarios.
809 * For more DBuf slices algorithm can get even more messy
810 * and less readable, so decided to use a table almost
811 * as is from BSpec itself - that way it is at least easier
812 * to compare, change and check.
813 */
814static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
815/* Autogenerated with igt/tools/intel_dbuf_map tool: */
816{
817 {
818 .active_pipes = BIT(PIPE_A),
819 .dbuf_mask = {
820 [PIPE_A] = BIT(DBUF_S1),
821 },
822 },
823 {
824 .active_pipes = BIT(PIPE_B),
825 .dbuf_mask = {
826 [PIPE_B] = BIT(DBUF_S1),
827 },
828 },
829 {
830 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
831 .dbuf_mask = {
832 [PIPE_A] = BIT(DBUF_S1),
833 [PIPE_B] = BIT(DBUF_S2),
834 },
835 },
836 {
837 .active_pipes = BIT(PIPE_C),
838 .dbuf_mask = {
839 [PIPE_C] = BIT(DBUF_S2),
840 },
841 },
842 {
843 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
844 .dbuf_mask = {
845 [PIPE_A] = BIT(DBUF_S1),
846 [PIPE_C] = BIT(DBUF_S2),
847 },
848 },
849 {
850 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
851 .dbuf_mask = {
852 [PIPE_B] = BIT(DBUF_S1),
853 [PIPE_C] = BIT(DBUF_S2),
854 },
855 },
856 {
857 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
858 .dbuf_mask = {
859 [PIPE_A] = BIT(DBUF_S1),
860 [PIPE_B] = BIT(DBUF_S1),
861 [PIPE_C] = BIT(DBUF_S2),
862 },
863 },
864 {}
865};
866
867/*
868 * Table taken from Bspec 49255
869 * Pipes do have some preferred DBuf slice affinity,
870 * plus there are some hardcoded requirements on how
871 * those should be distributed for multipipe scenarios.
872 * For more DBuf slices algorithm can get even more messy
873 * and less readable, so decided to use a table almost
874 * as is from BSpec itself - that way it is at least easier
875 * to compare, change and check.
876 */
877static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
878/* Autogenerated with igt/tools/intel_dbuf_map tool: */
879{
880 {
881 .active_pipes = BIT(PIPE_A),
882 .dbuf_mask = {
883 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
884 },
885 },
886 {
887 .active_pipes = BIT(PIPE_B),
888 .dbuf_mask = {
889 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
890 },
891 },
892 {
893 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
894 .dbuf_mask = {
895 [PIPE_A] = BIT(DBUF_S2),
896 [PIPE_B] = BIT(DBUF_S1),
897 },
898 },
899 {
900 .active_pipes = BIT(PIPE_C),
901 .dbuf_mask = {
902 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
903 },
904 },
905 {
906 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
907 .dbuf_mask = {
908 [PIPE_A] = BIT(DBUF_S1),
909 [PIPE_C] = BIT(DBUF_S2),
910 },
911 },
912 {
913 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
914 .dbuf_mask = {
915 [PIPE_B] = BIT(DBUF_S1),
916 [PIPE_C] = BIT(DBUF_S2),
917 },
918 },
919 {
920 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
921 .dbuf_mask = {
922 [PIPE_A] = BIT(DBUF_S1),
923 [PIPE_B] = BIT(DBUF_S1),
924 [PIPE_C] = BIT(DBUF_S2),
925 },
926 },
927 {
928 .active_pipes = BIT(PIPE_D),
929 .dbuf_mask = {
930 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
931 },
932 },
933 {
934 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
935 .dbuf_mask = {
936 [PIPE_A] = BIT(DBUF_S1),
937 [PIPE_D] = BIT(DBUF_S2),
938 },
939 },
940 {
941 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
942 .dbuf_mask = {
943 [PIPE_B] = BIT(DBUF_S1),
944 [PIPE_D] = BIT(DBUF_S2),
945 },
946 },
947 {
948 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
949 .dbuf_mask = {
950 [PIPE_A] = BIT(DBUF_S1),
951 [PIPE_B] = BIT(DBUF_S1),
952 [PIPE_D] = BIT(DBUF_S2),
953 },
954 },
955 {
956 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
957 .dbuf_mask = {
958 [PIPE_C] = BIT(DBUF_S1),
959 [PIPE_D] = BIT(DBUF_S2),
960 },
961 },
962 {
963 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
964 .dbuf_mask = {
965 [PIPE_A] = BIT(DBUF_S1),
966 [PIPE_C] = BIT(DBUF_S2),
967 [PIPE_D] = BIT(DBUF_S2),
968 },
969 },
970 {
971 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
972 .dbuf_mask = {
973 [PIPE_B] = BIT(DBUF_S1),
974 [PIPE_C] = BIT(DBUF_S2),
975 [PIPE_D] = BIT(DBUF_S2),
976 },
977 },
978 {
979 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
980 .dbuf_mask = {
981 [PIPE_A] = BIT(DBUF_S1),
982 [PIPE_B] = BIT(DBUF_S1),
983 [PIPE_C] = BIT(DBUF_S2),
984 [PIPE_D] = BIT(DBUF_S2),
985 },
986 },
987 {}
988};
989
990static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
991 {
992 .active_pipes = BIT(PIPE_A),
993 .dbuf_mask = {
994 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
995 },
996 },
997 {
998 .active_pipes = BIT(PIPE_B),
999 .dbuf_mask = {
1000 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1001 },
1002 },
1003 {
1004 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1005 .dbuf_mask = {
1006 [PIPE_A] = BIT(DBUF_S1),
1007 [PIPE_B] = BIT(DBUF_S2),
1008 },
1009 },
1010 {
1011 .active_pipes = BIT(PIPE_C),
1012 .dbuf_mask = {
1013 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1014 },
1015 },
1016 {
1017 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1018 .dbuf_mask = {
1019 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1020 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1021 },
1022 },
1023 {
1024 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1025 .dbuf_mask = {
1026 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1028 },
1029 },
1030 {
1031 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1032 .dbuf_mask = {
1033 [PIPE_A] = BIT(DBUF_S1),
1034 [PIPE_B] = BIT(DBUF_S2),
1035 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1036 },
1037 },
1038 {
1039 .active_pipes = BIT(PIPE_D),
1040 .dbuf_mask = {
1041 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1042 },
1043 },
1044 {
1045 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1046 .dbuf_mask = {
1047 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1048 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1049 },
1050 },
1051 {
1052 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1053 .dbuf_mask = {
1054 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1055 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1056 },
1057 },
1058 {
1059 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1060 .dbuf_mask = {
1061 [PIPE_A] = BIT(DBUF_S1),
1062 [PIPE_B] = BIT(DBUF_S2),
1063 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1064 },
1065 },
1066 {
1067 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1068 .dbuf_mask = {
1069 [PIPE_C] = BIT(DBUF_S3),
1070 [PIPE_D] = BIT(DBUF_S4),
1071 },
1072 },
1073 {
1074 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1075 .dbuf_mask = {
1076 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1077 [PIPE_C] = BIT(DBUF_S3),
1078 [PIPE_D] = BIT(DBUF_S4),
1079 },
1080 },
1081 {
1082 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1083 .dbuf_mask = {
1084 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1085 [PIPE_C] = BIT(DBUF_S3),
1086 [PIPE_D] = BIT(DBUF_S4),
1087 },
1088 },
1089 {
1090 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1091 .dbuf_mask = {
1092 [PIPE_A] = BIT(DBUF_S1),
1093 [PIPE_B] = BIT(DBUF_S2),
1094 [PIPE_C] = BIT(DBUF_S3),
1095 [PIPE_D] = BIT(DBUF_S4),
1096 },
1097 },
1098 {}
1099};
1100
1101static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1102 /*
1103 * Keep the join_mbus cases first so check_mbus_joined()
1104 * will prefer them over the !join_mbus cases.
1105 */
1106 {
1107 .active_pipes = BIT(PIPE_A),
1108 .dbuf_mask = {
1109 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1110 },
1111 .join_mbus = true,
1112 },
1113 {
1114 .active_pipes = BIT(PIPE_B),
1115 .dbuf_mask = {
1116 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1117 },
1118 .join_mbus = true,
1119 },
1120 {
1121 .active_pipes = BIT(PIPE_A),
1122 .dbuf_mask = {
1123 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1124 },
1125 .join_mbus = false,
1126 },
1127 {
1128 .active_pipes = BIT(PIPE_B),
1129 .dbuf_mask = {
1130 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1131 },
1132 .join_mbus = false,
1133 },
1134 {
1135 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1136 .dbuf_mask = {
1137 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1138 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1139 },
1140 },
1141 {
1142 .active_pipes = BIT(PIPE_C),
1143 .dbuf_mask = {
1144 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1145 },
1146 },
1147 {
1148 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1149 .dbuf_mask = {
1150 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1151 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1152 },
1153 },
1154 {
1155 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1156 .dbuf_mask = {
1157 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1158 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1159 },
1160 },
1161 {
1162 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1163 .dbuf_mask = {
1164 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1165 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1166 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1167 },
1168 },
1169 {
1170 .active_pipes = BIT(PIPE_D),
1171 .dbuf_mask = {
1172 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1173 },
1174 },
1175 {
1176 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1177 .dbuf_mask = {
1178 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1179 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1180 },
1181 },
1182 {
1183 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1184 .dbuf_mask = {
1185 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1186 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1187 },
1188 },
1189 {
1190 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1191 .dbuf_mask = {
1192 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1193 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1194 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1195 },
1196 },
1197 {
1198 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1199 .dbuf_mask = {
1200 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1201 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1202 },
1203 },
1204 {
1205 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1206 .dbuf_mask = {
1207 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1208 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1209 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1210 },
1211 },
1212 {
1213 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1214 .dbuf_mask = {
1215 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1216 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1217 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1218 },
1219 },
1220 {
1221 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1222 .dbuf_mask = {
1223 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1224 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1225 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1226 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1227 },
1228 },
1229 {}
1230
1231};
1232
1233static bool check_mbus_joined(u8 active_pipes,
1234 const struct dbuf_slice_conf_entry *dbuf_slices)
1235{
1236 int i;
1237
1238 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1239 if (dbuf_slices[i].active_pipes == active_pipes)
1240 return dbuf_slices[i].join_mbus;
1241 }
1242 return false;
1243}
1244
1245static bool adlp_check_mbus_joined(u8 active_pipes)
1246{
1247 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1248}
1249
1250static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1251 const struct dbuf_slice_conf_entry *dbuf_slices)
1252{
1253 int i;
1254
1255 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1256 if (dbuf_slices[i].active_pipes == active_pipes &&
1257 dbuf_slices[i].join_mbus == join_mbus)
1258 return dbuf_slices[i].dbuf_mask[pipe];
1259 }
1260 return 0;
1261}
1262
1263/*
1264 * This function finds an entry with same enabled pipe configuration and
1265 * returns correspondent DBuf slice mask as stated in BSpec for particular
1266 * platform.
1267 */
1268static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1269{
1270 /*
1271 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1272 * required calculating "pipe ratio" in order to determine
1273 * if one or two slices can be used for single pipe configurations
1274 * as additional constraint to the existing table.
1275 * However based on recent info, it should be not "pipe ratio"
1276 * but rather ratio between pixel_rate and cdclk with additional
1277 * constants, so for now we are using only table until this is
1278 * clarified. Also this is the reason why crtc_state param is
1279 * still here - we will need it once those additional constraints
1280 * pop up.
1281 */
1282 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1283 icl_allowed_dbufs);
1284}
1285
1286static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1287{
1288 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1289 tgl_allowed_dbufs);
1290}
1291
1292static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1293{
1294 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1295 adlp_allowed_dbufs);
1296}
1297
1298static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1299{
1300 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1301 dg2_allowed_dbufs);
1302}
1303
1304static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1305{
1306 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1307 enum pipe pipe = crtc->pipe;
1308
1309 if (IS_DG2(i915))
1310 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1311 else if (DISPLAY_VER(i915) >= 13)
1312 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1313 else if (DISPLAY_VER(i915) == 12)
1314 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1315 else if (DISPLAY_VER(i915) == 11)
1316 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1317 /*
1318 * For anything else just return one slice yet.
1319 * Should be extended for other platforms.
1320 */
1321 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1322}
1323
1324static bool
1325use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1326 struct intel_plane *plane)
1327{
1328 struct drm_i915_private *i915 = to_i915(plane->base.dev);
1329
1330 return DISPLAY_VER(i915) >= 13 &&
1331 crtc_state->uapi.async_flip &&
1332 plane->async_flip;
1333}
1334
1335static u64
1336skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1337{
1338 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1339 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1340 enum plane_id plane_id;
1341 u64 data_rate = 0;
1342
1343 for_each_plane_id_on_crtc(crtc, plane_id) {
1344 if (plane_id == PLANE_CURSOR)
1345 continue;
1346
1347 data_rate += crtc_state->rel_data_rate[plane_id];
1348
1349 if (DISPLAY_VER(i915) < 11)
1350 data_rate += crtc_state->rel_data_rate_y[plane_id];
1351 }
1352
1353 return data_rate;
1354}
1355
1356static const struct skl_wm_level *
1357skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1358 enum plane_id plane_id,
1359 int level)
1360{
1361 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1362
1363 if (level == 0 && pipe_wm->use_sagv_wm)
1364 return &wm->sagv.wm0;
1365
1366 return &wm->wm[level];
1367}
1368
1369static const struct skl_wm_level *
1370skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1371 enum plane_id plane_id)
1372{
1373 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1374
1375 if (pipe_wm->use_sagv_wm)
1376 return &wm->sagv.trans_wm;
1377
1378 return &wm->trans_wm;
1379}
1380
1381/*
1382 * We only disable the watermarks for each plane if
1383 * they exceed the ddb allocation of said plane. This
1384 * is done so that we don't end up touching cursor
1385 * watermarks needlessly when some other plane reduces
1386 * our max possible watermark level.
1387 *
1388 * Bspec has this to say about the PLANE_WM enable bit:
1389 * "All the watermarks at this level for all enabled
1390 * planes must be enabled before the level will be used."
1391 * So this is actually safe to do.
1392 */
1393static void
1394skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1395{
1396 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1397 memset(wm, 0, sizeof(*wm));
1398}
1399
1400static void
1401skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1402 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1403{
1404 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1405 uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1406 memset(wm, 0, sizeof(*wm));
1407 memset(uv_wm, 0, sizeof(*uv_wm));
1408 }
1409}
1410
1411static bool icl_need_wm1_wa(struct drm_i915_private *i915,
1412 enum plane_id plane_id)
1413{
1414 /*
1415 * Wa_1408961008:icl, ehl
1416 * Wa_14012656716:tgl, adl
1417 * Underruns with WM1+ disabled
1418 */
1419 return DISPLAY_VER(i915) == 11 ||
1420 (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
1421}
1422
1423struct skl_plane_ddb_iter {
1424 u64 data_rate;
1425 u16 start, size;
1426};
1427
1428static void
1429skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1430 struct skl_ddb_entry *ddb,
1431 const struct skl_wm_level *wm,
1432 u64 data_rate)
1433{
1434 u16 size, extra = 0;
1435
1436 if (data_rate) {
1437 extra = min_t(u16, iter->size,
1438 DIV64_U64_ROUND_UP(iter->size * data_rate,
1439 iter->data_rate));
1440 iter->size -= extra;
1441 iter->data_rate -= data_rate;
1442 }
1443
1444 /*
1445 * Keep ddb entry of all disabled planes explicitly zeroed
1446 * to avoid skl_ddb_add_affected_planes() adding them to
1447 * the state when other planes change their allocations.
1448 */
1449 size = wm->min_ddb_alloc + extra;
1450 if (size)
1451 iter->start = skl_ddb_entry_init(ddb, iter->start,
1452 iter->start + size);
1453}
1454
1455static int
1456skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1457 struct intel_crtc *crtc)
1458{
1459 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1460 struct intel_crtc_state *crtc_state =
1461 intel_atomic_get_new_crtc_state(state, crtc);
1462 const struct intel_dbuf_state *dbuf_state =
1463 intel_atomic_get_new_dbuf_state(state);
1464 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1465 int num_active = hweight8(dbuf_state->active_pipes);
1466 struct skl_plane_ddb_iter iter;
1467 enum plane_id plane_id;
1468 u16 cursor_size;
1469 u32 blocks;
1470 int level;
1471
1472 /* Clear the partitioning for disabled planes. */
1473 memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1474 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1475
1476 if (!crtc_state->hw.active)
1477 return 0;
1478
1479 iter.start = alloc->start;
1480 iter.size = skl_ddb_entry_size(alloc);
1481 if (iter.size == 0)
1482 return 0;
1483
1484 /* Allocate fixed number of blocks for cursor. */
1485 cursor_size = skl_cursor_allocation(crtc_state, num_active);
1486 iter.size -= cursor_size;
1487 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1488 alloc->end - cursor_size, alloc->end);
1489
1490 iter.data_rate = skl_total_relative_data_rate(crtc_state);
1491
1492 /*
1493 * Find the highest watermark level for which we can satisfy the block
1494 * requirement of active planes.
1495 */
1496 for (level = ilk_wm_max_level(i915); level >= 0; level--) {
1497 blocks = 0;
1498 for_each_plane_id_on_crtc(crtc, plane_id) {
1499 const struct skl_plane_wm *wm =
1500 &crtc_state->wm.skl.optimal.planes[plane_id];
1501
1502 if (plane_id == PLANE_CURSOR) {
1503 const struct skl_ddb_entry *ddb =
1504 &crtc_state->wm.skl.plane_ddb[plane_id];
1505
1506 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1507 drm_WARN_ON(&i915->drm,
1508 wm->wm[level].min_ddb_alloc != U16_MAX);
1509 blocks = U32_MAX;
1510 break;
1511 }
1512 continue;
1513 }
1514
1515 blocks += wm->wm[level].min_ddb_alloc;
1516 blocks += wm->uv_wm[level].min_ddb_alloc;
1517 }
1518
1519 if (blocks <= iter.size) {
1520 iter.size -= blocks;
1521 break;
1522 }
1523 }
1524
1525 if (level < 0) {
1526 drm_dbg_kms(&i915->drm,
1527 "Requested display configuration exceeds system DDB limitations");
1528 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1529 blocks, iter.size);
1530 return -EINVAL;
1531 }
1532
1533 /* avoid the WARN later when we don't allocate any extra DDB */
1534 if (iter.data_rate == 0)
1535 iter.size = 0;
1536
1537 /*
1538 * Grant each plane the blocks it requires at the highest achievable
1539 * watermark level, plus an extra share of the leftover blocks
1540 * proportional to its relative data rate.
1541 */
1542 for_each_plane_id_on_crtc(crtc, plane_id) {
1543 struct skl_ddb_entry *ddb =
1544 &crtc_state->wm.skl.plane_ddb[plane_id];
1545 struct skl_ddb_entry *ddb_y =
1546 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1547 const struct skl_plane_wm *wm =
1548 &crtc_state->wm.skl.optimal.planes[plane_id];
1549
1550 if (plane_id == PLANE_CURSOR)
1551 continue;
1552
1553 if (DISPLAY_VER(i915) < 11 &&
1554 crtc_state->nv12_planes & BIT(plane_id)) {
1555 skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1556 crtc_state->rel_data_rate_y[plane_id]);
1557 skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1558 crtc_state->rel_data_rate[plane_id]);
1559 } else {
1560 skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1561 crtc_state->rel_data_rate[plane_id]);
1562 }
1563 }
1564 drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1565
1566 /*
1567 * When we calculated watermark values we didn't know how high
1568 * of a level we'd actually be able to hit, so we just marked
1569 * all levels as "enabled." Go back now and disable the ones
1570 * that aren't actually possible.
1571 */
1572 for (level++; level <= ilk_wm_max_level(i915); level++) {
1573 for_each_plane_id_on_crtc(crtc, plane_id) {
1574 const struct skl_ddb_entry *ddb =
1575 &crtc_state->wm.skl.plane_ddb[plane_id];
1576 const struct skl_ddb_entry *ddb_y =
1577 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1578 struct skl_plane_wm *wm =
1579 &crtc_state->wm.skl.optimal.planes[plane_id];
1580
1581 if (DISPLAY_VER(i915) < 11 &&
1582 crtc_state->nv12_planes & BIT(plane_id))
1583 skl_check_nv12_wm_level(&wm->wm[level],
1584 &wm->uv_wm[level],
1585 ddb_y, ddb);
1586 else
1587 skl_check_wm_level(&wm->wm[level], ddb);
1588
1589 if (icl_need_wm1_wa(i915, plane_id) &&
1590 level == 1 && !wm->wm[level].enable &&
1591 wm->wm[0].enable) {
1592 wm->wm[level].blocks = wm->wm[0].blocks;
1593 wm->wm[level].lines = wm->wm[0].lines;
1594 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
1595 }
1596 }
1597 }
1598
1599 /*
1600 * Go back and disable the transition and SAGV watermarks
1601 * if it turns out we don't have enough DDB blocks for them.
1602 */
1603 for_each_plane_id_on_crtc(crtc, plane_id) {
1604 const struct skl_ddb_entry *ddb =
1605 &crtc_state->wm.skl.plane_ddb[plane_id];
1606 const struct skl_ddb_entry *ddb_y =
1607 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1608 struct skl_plane_wm *wm =
1609 &crtc_state->wm.skl.optimal.planes[plane_id];
1610
1611 if (DISPLAY_VER(i915) < 11 &&
1612 crtc_state->nv12_planes & BIT(plane_id)) {
1613 skl_check_wm_level(&wm->trans_wm, ddb_y);
1614 } else {
1615 WARN_ON(skl_ddb_entry_size(ddb_y));
1616
1617 skl_check_wm_level(&wm->trans_wm, ddb);
1618 }
1619
1620 skl_check_wm_level(&wm->sagv.wm0, ddb);
1621 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1622 }
1623
1624 return 0;
1625}
1626
1627/*
1628 * The max latency should be 257 (max the punit can code is 255 and we add 2us
1629 * for the read latency) and cpp should always be <= 8, so that
1630 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1631 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1632 */
1633static uint_fixed_16_16_t
1634skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1635 u8 cpp, u32 latency, u32 dbuf_block_size)
1636{
1637 u32 wm_intermediate_val;
1638 uint_fixed_16_16_t ret;
1639
1640 if (latency == 0)
1641 return FP_16_16_MAX;
1642
1643 wm_intermediate_val = latency * pixel_rate * cpp;
1644 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1645
1646 if (DISPLAY_VER(i915) >= 10)
1647 ret = add_fixed16_u32(ret, 1);
1648
1649 return ret;
1650}
1651
1652static uint_fixed_16_16_t
1653skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1654 uint_fixed_16_16_t plane_blocks_per_line)
1655{
1656 u32 wm_intermediate_val;
1657 uint_fixed_16_16_t ret;
1658
1659 if (latency == 0)
1660 return FP_16_16_MAX;
1661
1662 wm_intermediate_val = latency * pixel_rate;
1663 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1664 pipe_htotal * 1000);
1665 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1666 return ret;
1667}
1668
1669static uint_fixed_16_16_t
1670intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1671{
1672 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1673 u32 pixel_rate;
1674 u32 crtc_htotal;
1675 uint_fixed_16_16_t linetime_us;
1676
1677 if (!crtc_state->hw.active)
1678 return u32_to_fixed16(0);
1679
1680 pixel_rate = crtc_state->pixel_rate;
1681
1682 if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1683 return u32_to_fixed16(0);
1684
1685 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1686 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1687
1688 return linetime_us;
1689}
1690
1691static int
1692skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1693 int width, const struct drm_format_info *format,
1694 u64 modifier, unsigned int rotation,
1695 u32 plane_pixel_rate, struct skl_wm_params *wp,
1696 int color_plane)
1697{
1698 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1699 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1700 u32 interm_pbpl;
1701
1702 /* only planar format has two planes */
1703 if (color_plane == 1 &&
1704 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1705 drm_dbg_kms(&i915->drm,
1706 "Non planar format have single plane\n");
1707 return -EINVAL;
1708 }
1709
1710 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1711 wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1712 intel_fb_is_tiled_modifier(modifier);
1713 wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1714 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1715
1716 wp->width = width;
1717 if (color_plane == 1 && wp->is_planar)
1718 wp->width /= 2;
1719
1720 wp->cpp = format->cpp[color_plane];
1721 wp->plane_pixel_rate = plane_pixel_rate;
1722
1723 if (DISPLAY_VER(i915) >= 11 &&
1724 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1725 wp->dbuf_block_size = 256;
1726 else
1727 wp->dbuf_block_size = 512;
1728
1729 if (drm_rotation_90_or_270(rotation)) {
1730 switch (wp->cpp) {
1731 case 1:
1732 wp->y_min_scanlines = 16;
1733 break;
1734 case 2:
1735 wp->y_min_scanlines = 8;
1736 break;
1737 case 4:
1738 wp->y_min_scanlines = 4;
1739 break;
1740 default:
1741 MISSING_CASE(wp->cpp);
1742 return -EINVAL;
1743 }
1744 } else {
1745 wp->y_min_scanlines = 4;
1746 }
1747
1748 if (skl_needs_memory_bw_wa(i915))
1749 wp->y_min_scanlines *= 2;
1750
1751 wp->plane_bytes_per_line = wp->width * wp->cpp;
1752 if (wp->y_tiled) {
1753 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1754 wp->y_min_scanlines,
1755 wp->dbuf_block_size);
1756
1757 if (DISPLAY_VER(i915) >= 10)
1758 interm_pbpl++;
1759
1760 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1761 wp->y_min_scanlines);
1762 } else {
1763 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1764 wp->dbuf_block_size);
1765
1766 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1767 interm_pbpl++;
1768
1769 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1770 }
1771
1772 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1773 wp->plane_blocks_per_line);
1774
1775 wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1776
1777 return 0;
1778}
1779
1780static int
1781skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1782 const struct intel_plane_state *plane_state,
1783 struct skl_wm_params *wp, int color_plane)
1784{
1785 const struct drm_framebuffer *fb = plane_state->hw.fb;
1786 int width;
1787
1788 /*
1789 * Src coordinates are already rotated by 270 degrees for
1790 * the 90/270 degree plane rotation cases (to match the
1791 * GTT mapping), hence no need to account for rotation here.
1792 */
1793 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1794
1795 return skl_compute_wm_params(crtc_state, width,
1796 fb->format, fb->modifier,
1797 plane_state->hw.rotation,
1798 intel_plane_pixel_rate(crtc_state, plane_state),
1799 wp, color_plane);
1800}
1801
1802static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1803{
1804 if (DISPLAY_VER(i915) >= 10)
1805 return true;
1806
1807 /* The number of lines are ignored for the level 0 watermark. */
1808 return level > 0;
1809}
1810
1811static int skl_wm_max_lines(struct drm_i915_private *i915)
1812{
1813 if (DISPLAY_VER(i915) >= 13)
1814 return 255;
1815 else
1816 return 31;
1817}
1818
1819static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1820 struct intel_plane *plane,
1821 int level,
1822 unsigned int latency,
1823 const struct skl_wm_params *wp,
1824 const struct skl_wm_level *result_prev,
1825 struct skl_wm_level *result /* out */)
1826{
1827 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1828 uint_fixed_16_16_t method1, method2;
1829 uint_fixed_16_16_t selected_result;
1830 u32 blocks, lines, min_ddb_alloc = 0;
1831
1832 if (latency == 0 ||
1833 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1834 /* reject it */
1835 result->min_ddb_alloc = U16_MAX;
1836 return;
1837 }
1838
1839 /*
1840 * WaIncreaseLatencyIPCEnabled: kbl,cfl
1841 * Display WA #1141: kbl,cfl
1842 */
1843 if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
1844 skl_watermark_ipc_enabled(i915))
1845 latency += 4;
1846
1847 if (skl_needs_memory_bw_wa(i915) && wp->x_tiled)
1848 latency += 15;
1849
1850 method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1851 wp->cpp, latency, wp->dbuf_block_size);
1852 method2 = skl_wm_method2(wp->plane_pixel_rate,
1853 crtc_state->hw.pipe_mode.crtc_htotal,
1854 latency,
1855 wp->plane_blocks_per_line);
1856
1857 if (wp->y_tiled) {
1858 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1859 } else {
1860 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1861 wp->dbuf_block_size < 1) &&
1862 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1863 selected_result = method2;
1864 } else if (latency >= wp->linetime_us) {
1865 if (DISPLAY_VER(i915) == 9)
1866 selected_result = min_fixed16(method1, method2);
1867 else
1868 selected_result = method2;
1869 } else {
1870 selected_result = method1;
1871 }
1872 }
1873
1874 blocks = fixed16_to_u32_round_up(selected_result) + 1;
1875 /*
1876 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1877 * as there will be at minimum one line for lines configuration. This
1878 * is a work around for FIFO underruns observed with resolutions like
1879 * 4k 60 Hz in single channel DRAM configurations.
1880 *
1881 * As per the Bspec 49325, if the ddb allocation can hold at least
1882 * one plane_blocks_per_line, we should have selected method2 in
1883 * the above logic. Assuming that modern versions have enough dbuf
1884 * and method2 guarantees blocks equivalent to at least 1 line,
1885 * select the blocks as plane_blocks_per_line.
1886 *
1887 * TODO: Revisit the logic when we have better understanding on DRAM
1888 * channels' impact on the level 0 memory latency and the relevant
1889 * wm calculations.
1890 */
1891 if (skl_wm_has_lines(i915, level))
1892 blocks = max(blocks,
1893 fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1894 lines = div_round_up_fixed16(selected_result,
1895 wp->plane_blocks_per_line);
1896
1897 if (DISPLAY_VER(i915) == 9) {
1898 /* Display WA #1125: skl,bxt,kbl */
1899 if (level == 0 && wp->rc_surface)
1900 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1901
1902 /* Display WA #1126: skl,bxt,kbl */
1903 if (level >= 1 && level <= 7) {
1904 if (wp->y_tiled) {
1905 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1906 lines += wp->y_min_scanlines;
1907 } else {
1908 blocks++;
1909 }
1910
1911 /*
1912 * Make sure result blocks for higher latency levels are
1913 * at least as high as level below the current level.
1914 * Assumption in DDB algorithm optimization for special
1915 * cases. Also covers Display WA #1125 for RC.
1916 */
1917 if (result_prev->blocks > blocks)
1918 blocks = result_prev->blocks;
1919 }
1920 }
1921
1922 if (DISPLAY_VER(i915) >= 11) {
1923 if (wp->y_tiled) {
1924 int extra_lines;
1925
1926 if (lines % wp->y_min_scanlines == 0)
1927 extra_lines = wp->y_min_scanlines;
1928 else
1929 extra_lines = wp->y_min_scanlines * 2 -
1930 lines % wp->y_min_scanlines;
1931
1932 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1933 wp->plane_blocks_per_line);
1934 } else {
1935 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1936 }
1937 }
1938
1939 if (!skl_wm_has_lines(i915, level))
1940 lines = 0;
1941
1942 if (lines > skl_wm_max_lines(i915)) {
1943 /* reject it */
1944 result->min_ddb_alloc = U16_MAX;
1945 return;
1946 }
1947
1948 /*
1949 * If lines is valid, assume we can use this watermark level
1950 * for now. We'll come back and disable it after we calculate the
1951 * DDB allocation if it turns out we don't actually have enough
1952 * blocks to satisfy it.
1953 */
1954 result->blocks = blocks;
1955 result->lines = lines;
1956 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1957 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1958 result->enable = true;
1959
1960 if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1961 result->can_sagv = latency >= i915->display.sagv.block_time_us;
1962}
1963
1964static void
1965skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1966 struct intel_plane *plane,
1967 const struct skl_wm_params *wm_params,
1968 struct skl_wm_level *levels)
1969{
1970 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1971 int level, max_level = ilk_wm_max_level(i915);
1972 struct skl_wm_level *result_prev = &levels[0];
1973
1974 for (level = 0; level <= max_level; level++) {
1975 struct skl_wm_level *result = &levels[level];
1976 unsigned int latency = i915->display.wm.skl_latency[level];
1977
1978 skl_compute_plane_wm(crtc_state, plane, level, latency,
1979 wm_params, result_prev, result);
1980
1981 result_prev = result;
1982 }
1983}
1984
1985static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
1986 struct intel_plane *plane,
1987 const struct skl_wm_params *wm_params,
1988 struct skl_plane_wm *plane_wm)
1989{
1990 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1991 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
1992 struct skl_wm_level *levels = plane_wm->wm;
1993 unsigned int latency = 0;
1994
1995 if (i915->display.sagv.block_time_us)
1996 latency = i915->display.sagv.block_time_us + i915->display.wm.skl_latency[0];
1997
1998 skl_compute_plane_wm(crtc_state, plane, 0, latency,
1999 wm_params, &levels[0],
2000 sagv_wm);
2001}
2002
2003static void skl_compute_transition_wm(struct drm_i915_private *i915,
2004 struct skl_wm_level *trans_wm,
2005 const struct skl_wm_level *wm0,
2006 const struct skl_wm_params *wp)
2007{
2008 u16 trans_min, trans_amount, trans_y_tile_min;
2009 u16 wm0_blocks, trans_offset, blocks;
2010
2011 /* Transition WM don't make any sense if ipc is disabled */
2012 if (!skl_watermark_ipc_enabled(i915))
2013 return;
2014
2015 /*
2016 * WaDisableTWM:skl,kbl,cfl,bxt
2017 * Transition WM are not recommended by HW team for GEN9
2018 */
2019 if (DISPLAY_VER(i915) == 9)
2020 return;
2021
2022 if (DISPLAY_VER(i915) >= 11)
2023 trans_min = 4;
2024 else
2025 trans_min = 14;
2026
2027 /* Display WA #1140: glk,cnl */
2028 if (DISPLAY_VER(i915) == 10)
2029 trans_amount = 0;
2030 else
2031 trans_amount = 10; /* This is configurable amount */
2032
2033 trans_offset = trans_min + trans_amount;
2034
2035 /*
2036 * The spec asks for Selected Result Blocks for wm0 (the real value),
2037 * not Result Blocks (the integer value). Pay attention to the capital
2038 * letters. The value wm_l0->blocks is actually Result Blocks, but
2039 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2040 * and since we later will have to get the ceiling of the sum in the
2041 * transition watermarks calculation, we can just pretend Selected
2042 * Result Blocks is Result Blocks minus 1 and it should work for the
2043 * current platforms.
2044 */
2045 wm0_blocks = wm0->blocks - 1;
2046
2047 if (wp->y_tiled) {
2048 trans_y_tile_min =
2049 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2050 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2051 } else {
2052 blocks = wm0_blocks + trans_offset;
2053 }
2054 blocks++;
2055
2056 /*
2057 * Just assume we can enable the transition watermark. After
2058 * computing the DDB we'll come back and disable it if that
2059 * assumption turns out to be false.
2060 */
2061 trans_wm->blocks = blocks;
2062 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2063 trans_wm->enable = true;
2064}
2065
2066static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2067 const struct intel_plane_state *plane_state,
2068 struct intel_plane *plane, int color_plane)
2069{
2070 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2071 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2072 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2073 struct skl_wm_params wm_params;
2074 int ret;
2075
2076 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2077 &wm_params, color_plane);
2078 if (ret)
2079 return ret;
2080
2081 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2082
2083 skl_compute_transition_wm(i915, &wm->trans_wm,
2084 &wm->wm[0], &wm_params);
2085
2086 if (DISPLAY_VER(i915) >= 12) {
2087 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2088
2089 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2090 &wm->sagv.wm0, &wm_params);
2091 }
2092
2093 return 0;
2094}
2095
2096static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2097 const struct intel_plane_state *plane_state,
2098 struct intel_plane *plane)
2099{
2100 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2101 struct skl_wm_params wm_params;
2102 int ret;
2103
2104 wm->is_planar = true;
2105
2106 /* uv plane watermarks must also be validated for NV12/Planar */
2107 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2108 &wm_params, 1);
2109 if (ret)
2110 return ret;
2111
2112 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2113
2114 return 0;
2115}
2116
2117static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2118 const struct intel_plane_state *plane_state)
2119{
2120 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2121 enum plane_id plane_id = plane->id;
2122 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2123 const struct drm_framebuffer *fb = plane_state->hw.fb;
2124 int ret;
2125
2126 memset(wm, 0, sizeof(*wm));
2127
2128 if (!intel_wm_plane_visible(crtc_state, plane_state))
2129 return 0;
2130
2131 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2132 plane, 0);
2133 if (ret)
2134 return ret;
2135
2136 if (fb->format->is_yuv && fb->format->num_planes > 1) {
2137 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2138 plane);
2139 if (ret)
2140 return ret;
2141 }
2142
2143 return 0;
2144}
2145
2146static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2147 const struct intel_plane_state *plane_state)
2148{
2149 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2150 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2151 enum plane_id plane_id = plane->id;
2152 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2153 int ret;
2154
2155 /* Watermarks calculated in master */
2156 if (plane_state->planar_slave)
2157 return 0;
2158
2159 memset(wm, 0, sizeof(*wm));
2160
2161 if (plane_state->planar_linked_plane) {
2162 const struct drm_framebuffer *fb = plane_state->hw.fb;
2163
2164 drm_WARN_ON(&i915->drm,
2165 !intel_wm_plane_visible(crtc_state, plane_state));
2166 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2167 fb->format->num_planes == 1);
2168
2169 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2170 plane_state->planar_linked_plane, 0);
2171 if (ret)
2172 return ret;
2173
2174 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2175 plane, 1);
2176 if (ret)
2177 return ret;
2178 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2179 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2180 plane, 0);
2181 if (ret)
2182 return ret;
2183 }
2184
2185 return 0;
2186}
2187
2188static int skl_build_pipe_wm(struct intel_atomic_state *state,
2189 struct intel_crtc *crtc)
2190{
2191 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2192 struct intel_crtc_state *crtc_state =
2193 intel_atomic_get_new_crtc_state(state, crtc);
2194 const struct intel_plane_state *plane_state;
2195 struct intel_plane *plane;
2196 int ret, i;
2197
2198 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2199 /*
2200 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2201 * instead but we don't populate that correctly for NV12 Y
2202 * planes so for now hack this.
2203 */
2204 if (plane->pipe != crtc->pipe)
2205 continue;
2206
2207 if (DISPLAY_VER(i915) >= 11)
2208 ret = icl_build_plane_wm(crtc_state, plane_state);
2209 else
2210 ret = skl_build_plane_wm(crtc_state, plane_state);
2211 if (ret)
2212 return ret;
2213 }
2214
2215 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2216
2217 return 0;
2218}
2219
2220static void skl_ddb_entry_write(struct drm_i915_private *i915,
2221 i915_reg_t reg,
2222 const struct skl_ddb_entry *entry)
2223{
2224 if (entry->end)
2225 intel_de_write_fw(i915, reg,
2226 PLANE_BUF_END(entry->end - 1) |
2227 PLANE_BUF_START(entry->start));
2228 else
2229 intel_de_write_fw(i915, reg, 0);
2230}
2231
2232static void skl_write_wm_level(struct drm_i915_private *i915,
2233 i915_reg_t reg,
2234 const struct skl_wm_level *level)
2235{
2236 u32 val = 0;
2237
2238 if (level->enable)
2239 val |= PLANE_WM_EN;
2240 if (level->ignore_lines)
2241 val |= PLANE_WM_IGNORE_LINES;
2242 val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2243 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2244
2245 intel_de_write_fw(i915, reg, val);
2246}
2247
2248void skl_write_plane_wm(struct intel_plane *plane,
2249 const struct intel_crtc_state *crtc_state)
2250{
2251 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2252 int level, max_level = ilk_wm_max_level(i915);
2253 enum plane_id plane_id = plane->id;
2254 enum pipe pipe = plane->pipe;
2255 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2256 const struct skl_ddb_entry *ddb =
2257 &crtc_state->wm.skl.plane_ddb[plane_id];
2258 const struct skl_ddb_entry *ddb_y =
2259 &crtc_state->wm.skl.plane_ddb_y[plane_id];
2260
2261 for (level = 0; level <= max_level; level++)
2262 skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2263 skl_plane_wm_level(pipe_wm, plane_id, level));
2264
2265 skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2266 skl_plane_trans_wm(pipe_wm, plane_id));
2267
2268 if (HAS_HW_SAGV_WM(i915)) {
2269 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2270
2271 skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2272 &wm->sagv.wm0);
2273 skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2274 &wm->sagv.trans_wm);
2275 }
2276
2277 skl_ddb_entry_write(i915,
2278 PLANE_BUF_CFG(pipe, plane_id), ddb);
2279
2280 if (DISPLAY_VER(i915) < 11)
2281 skl_ddb_entry_write(i915,
2282 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
2283}
2284
2285void skl_write_cursor_wm(struct intel_plane *plane,
2286 const struct intel_crtc_state *crtc_state)
2287{
2288 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2289 int level, max_level = ilk_wm_max_level(i915);
2290 enum plane_id plane_id = plane->id;
2291 enum pipe pipe = plane->pipe;
2292 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2293 const struct skl_ddb_entry *ddb =
2294 &crtc_state->wm.skl.plane_ddb[plane_id];
2295
2296 for (level = 0; level <= max_level; level++)
2297 skl_write_wm_level(i915, CUR_WM(pipe, level),
2298 skl_plane_wm_level(pipe_wm, plane_id, level));
2299
2300 skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2301 skl_plane_trans_wm(pipe_wm, plane_id));
2302
2303 if (HAS_HW_SAGV_WM(i915)) {
2304 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2305
2306 skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2307 &wm->sagv.wm0);
2308 skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2309 &wm->sagv.trans_wm);
2310 }
2311
2312 skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
2313}
2314
2315static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2316 const struct skl_wm_level *l2)
2317{
2318 return l1->enable == l2->enable &&
2319 l1->ignore_lines == l2->ignore_lines &&
2320 l1->lines == l2->lines &&
2321 l1->blocks == l2->blocks;
2322}
2323
2324static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2325 const struct skl_plane_wm *wm1,
2326 const struct skl_plane_wm *wm2)
2327{
2328 int level, max_level = ilk_wm_max_level(i915);
2329
2330 for (level = 0; level <= max_level; level++) {
2331 /*
2332 * We don't check uv_wm as the hardware doesn't actually
2333 * use it. It only gets used for calculating the required
2334 * ddb allocation.
2335 */
2336 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2337 return false;
2338 }
2339
2340 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2341 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2342 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2343}
2344
2345static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2346 const struct skl_ddb_entry *b)
2347{
2348 return a->start < b->end && b->start < a->end;
2349}
2350
2351static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2352 const struct skl_ddb_entry *b)
2353{
2354 if (a->end && b->end) {
2355 a->start = min(a->start, b->start);
2356 a->end = max(a->end, b->end);
2357 } else if (b->end) {
2358 a->start = b->start;
2359 a->end = b->end;
2360 }
2361}
2362
2363bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2364 const struct skl_ddb_entry *entries,
2365 int num_entries, int ignore_idx)
2366{
2367 int i;
2368
2369 for (i = 0; i < num_entries; i++) {
2370 if (i != ignore_idx &&
2371 skl_ddb_entries_overlap(ddb, &entries[i]))
2372 return true;
2373 }
2374
2375 return false;
2376}
2377
2378static int
2379skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2380 struct intel_crtc_state *new_crtc_state)
2381{
2382 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2383 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2384 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2385 struct intel_plane *plane;
2386
2387 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2388 struct intel_plane_state *plane_state;
2389 enum plane_id plane_id = plane->id;
2390
2391 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2392 &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2393 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2394 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2395 continue;
2396
2397 plane_state = intel_atomic_get_plane_state(state, plane);
2398 if (IS_ERR(plane_state))
2399 return PTR_ERR(plane_state);
2400
2401 new_crtc_state->update_planes |= BIT(plane_id);
2402 }
2403
2404 return 0;
2405}
2406
2407static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2408{
2409 struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2410 u8 enabled_slices;
2411 enum pipe pipe;
2412
2413 /*
2414 * FIXME: For now we always enable slice S1 as per
2415 * the Bspec display initialization sequence.
2416 */
2417 enabled_slices = BIT(DBUF_S1);
2418
2419 for_each_pipe(i915, pipe)
2420 enabled_slices |= dbuf_state->slices[pipe];
2421
2422 return enabled_slices;
2423}
2424
2425static int
2426skl_compute_ddb(struct intel_atomic_state *state)
2427{
2428 struct drm_i915_private *i915 = to_i915(state->base.dev);
2429 const struct intel_dbuf_state *old_dbuf_state;
2430 struct intel_dbuf_state *new_dbuf_state = NULL;
2431 const struct intel_crtc_state *old_crtc_state;
2432 struct intel_crtc_state *new_crtc_state;
2433 struct intel_crtc *crtc;
2434 int ret, i;
2435
2436 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2437 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2438 if (IS_ERR(new_dbuf_state))
2439 return PTR_ERR(new_dbuf_state);
2440
2441 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2442 break;
2443 }
2444
2445 if (!new_dbuf_state)
2446 return 0;
2447
2448 new_dbuf_state->active_pipes =
2449 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2450
2451 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2452 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2453 if (ret)
2454 return ret;
2455 }
2456
2457 if (HAS_MBUS_JOINING(i915))
2458 new_dbuf_state->joined_mbus =
2459 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2460
2461 for_each_intel_crtc(&i915->drm, crtc) {
2462 enum pipe pipe = crtc->pipe;
2463
2464 new_dbuf_state->slices[pipe] =
2465 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2466 new_dbuf_state->joined_mbus);
2467
2468 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2469 continue;
2470
2471 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2472 if (ret)
2473 return ret;
2474 }
2475
2476 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2477
2478 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2479 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2480 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2481 if (ret)
2482 return ret;
2483
2484 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2485 /* TODO: Implement vblank synchronized MBUS joining changes */
2486 ret = intel_modeset_all_pipes(state, "MBUS joining change");
2487 if (ret)
2488 return ret;
2489 }
2490
2491 drm_dbg_kms(&i915->drm,
2492 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2493 old_dbuf_state->enabled_slices,
2494 new_dbuf_state->enabled_slices,
2495 INTEL_INFO(i915)->display.dbuf.slice_mask,
2496 str_yes_no(old_dbuf_state->joined_mbus),
2497 str_yes_no(new_dbuf_state->joined_mbus));
2498 }
2499
2500 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2501 enum pipe pipe = crtc->pipe;
2502
2503 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2504
2505 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2506 continue;
2507
2508 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2509 if (ret)
2510 return ret;
2511 }
2512
2513 for_each_intel_crtc(&i915->drm, crtc) {
2514 ret = skl_crtc_allocate_ddb(state, crtc);
2515 if (ret)
2516 return ret;
2517 }
2518
2519 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2520 new_crtc_state, i) {
2521 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2522 if (ret)
2523 return ret;
2524
2525 ret = skl_ddb_add_affected_planes(old_crtc_state,
2526 new_crtc_state);
2527 if (ret)
2528 return ret;
2529 }
2530
2531 return 0;
2532}
2533
2534static char enast(bool enable)
2535{
2536 return enable ? '*' : ' ';
2537}
2538
2539static void
2540skl_print_wm_changes(struct intel_atomic_state *state)
2541{
2542 struct drm_i915_private *i915 = to_i915(state->base.dev);
2543 const struct intel_crtc_state *old_crtc_state;
2544 const struct intel_crtc_state *new_crtc_state;
2545 struct intel_plane *plane;
2546 struct intel_crtc *crtc;
2547 int i;
2548
2549 if (!drm_debug_enabled(DRM_UT_KMS))
2550 return;
2551
2552 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2553 new_crtc_state, i) {
2554 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2555
2556 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2557 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2558
2559 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2560 enum plane_id plane_id = plane->id;
2561 const struct skl_ddb_entry *old, *new;
2562
2563 old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2564 new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2565
2566 if (skl_ddb_entry_equal(old, new))
2567 continue;
2568
2569 drm_dbg_kms(&i915->drm,
2570 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2571 plane->base.base.id, plane->base.name,
2572 old->start, old->end, new->start, new->end,
2573 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2574 }
2575
2576 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2577 enum plane_id plane_id = plane->id;
2578 const struct skl_plane_wm *old_wm, *new_wm;
2579
2580 old_wm = &old_pipe_wm->planes[plane_id];
2581 new_wm = &new_pipe_wm->planes[plane_id];
2582
2583 if (skl_plane_wm_equals(i915, old_wm, new_wm))
2584 continue;
2585
2586 drm_dbg_kms(&i915->drm,
2587 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2588 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2589 plane->base.base.id, plane->base.name,
2590 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2591 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2592 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2593 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2594 enast(old_wm->trans_wm.enable),
2595 enast(old_wm->sagv.wm0.enable),
2596 enast(old_wm->sagv.trans_wm.enable),
2597 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2598 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2599 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2600 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2601 enast(new_wm->trans_wm.enable),
2602 enast(new_wm->sagv.wm0.enable),
2603 enast(new_wm->sagv.trans_wm.enable));
2604
2605 drm_dbg_kms(&i915->drm,
2606 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2607 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2608 plane->base.base.id, plane->base.name,
2609 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2610 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2611 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2612 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2613 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2614 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2615 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2616 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2617 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2618 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2619 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2620 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2621 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2622 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2623 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2624 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2625 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2626 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2627 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2628 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2629 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2630 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2631
2632 drm_dbg_kms(&i915->drm,
2633 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2634 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2635 plane->base.base.id, plane->base.name,
2636 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2637 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2638 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2639 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2640 old_wm->trans_wm.blocks,
2641 old_wm->sagv.wm0.blocks,
2642 old_wm->sagv.trans_wm.blocks,
2643 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2644 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2645 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2646 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2647 new_wm->trans_wm.blocks,
2648 new_wm->sagv.wm0.blocks,
2649 new_wm->sagv.trans_wm.blocks);
2650
2651 drm_dbg_kms(&i915->drm,
2652 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2653 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2654 plane->base.base.id, plane->base.name,
2655 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2656 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2657 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2658 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2659 old_wm->trans_wm.min_ddb_alloc,
2660 old_wm->sagv.wm0.min_ddb_alloc,
2661 old_wm->sagv.trans_wm.min_ddb_alloc,
2662 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2663 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2664 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2665 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2666 new_wm->trans_wm.min_ddb_alloc,
2667 new_wm->sagv.wm0.min_ddb_alloc,
2668 new_wm->sagv.trans_wm.min_ddb_alloc);
2669 }
2670 }
2671}
2672
2673static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2674 const struct skl_pipe_wm *old_pipe_wm,
2675 const struct skl_pipe_wm *new_pipe_wm)
2676{
2677 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2678 int level, max_level = ilk_wm_max_level(i915);
2679
2680 for (level = 0; level <= max_level; level++) {
2681 /*
2682 * We don't check uv_wm as the hardware doesn't actually
2683 * use it. It only gets used for calculating the required
2684 * ddb allocation.
2685 */
2686 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2687 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2688 return false;
2689 }
2690
2691 if (HAS_HW_SAGV_WM(i915)) {
2692 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2693 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2694
2695 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2696 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2697 return false;
2698 }
2699
2700 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2701 skl_plane_trans_wm(new_pipe_wm, plane->id));
2702}
2703
2704/*
2705 * To make sure the cursor watermark registers are always consistent
2706 * with our computed state the following scenario needs special
2707 * treatment:
2708 *
2709 * 1. enable cursor
2710 * 2. move cursor entirely offscreen
2711 * 3. disable cursor
2712 *
2713 * Step 2. does call .disable_plane() but does not zero the watermarks
2714 * (since we consider an offscreen cursor still active for the purposes
2715 * of watermarks). Step 3. would not normally call .disable_plane()
2716 * because the actual plane visibility isn't changing, and we don't
2717 * deallocate the cursor ddb until the pipe gets disabled. So we must
2718 * force step 3. to call .disable_plane() to update the watermark
2719 * registers properly.
2720 *
2721 * Other planes do not suffer from this issues as their watermarks are
2722 * calculated based on the actual plane visibility. The only time this
2723 * can trigger for the other planes is during the initial readout as the
2724 * default value of the watermarks registers is not zero.
2725 */
2726static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2727 struct intel_crtc *crtc)
2728{
2729 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2730 const struct intel_crtc_state *old_crtc_state =
2731 intel_atomic_get_old_crtc_state(state, crtc);
2732 struct intel_crtc_state *new_crtc_state =
2733 intel_atomic_get_new_crtc_state(state, crtc);
2734 struct intel_plane *plane;
2735
2736 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2737 struct intel_plane_state *plane_state;
2738 enum plane_id plane_id = plane->id;
2739
2740 /*
2741 * Force a full wm update for every plane on modeset.
2742 * Required because the reset value of the wm registers
2743 * is non-zero, whereas we want all disabled planes to
2744 * have zero watermarks. So if we turn off the relevant
2745 * power well the hardware state will go out of sync
2746 * with the software state.
2747 */
2748 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2749 skl_plane_selected_wm_equals(plane,
2750 &old_crtc_state->wm.skl.optimal,
2751 &new_crtc_state->wm.skl.optimal))
2752 continue;
2753
2754 plane_state = intel_atomic_get_plane_state(state, plane);
2755 if (IS_ERR(plane_state))
2756 return PTR_ERR(plane_state);
2757
2758 new_crtc_state->update_planes |= BIT(plane_id);
2759 }
2760
2761 return 0;
2762}
2763
2764static int
2765skl_compute_wm(struct intel_atomic_state *state)
2766{
2767 struct intel_crtc *crtc;
2768 struct intel_crtc_state *new_crtc_state;
2769 int ret, i;
2770
2771 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2772 ret = skl_build_pipe_wm(state, crtc);
2773 if (ret)
2774 return ret;
2775 }
2776
2777 ret = skl_compute_ddb(state);
2778 if (ret)
2779 return ret;
2780
2781 ret = intel_compute_sagv_mask(state);
2782 if (ret)
2783 return ret;
2784
2785 /*
2786 * skl_compute_ddb() will have adjusted the final watermarks
2787 * based on how much ddb is available. Now we can actually
2788 * check if the final watermarks changed.
2789 */
2790 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2791 ret = skl_wm_add_affected_planes(state, crtc);
2792 if (ret)
2793 return ret;
2794 }
2795
2796 skl_print_wm_changes(state);
2797
2798 return 0;
2799}
2800
2801static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2802{
2803 level->enable = val & PLANE_WM_EN;
2804 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2805 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2806 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2807}
2808
2809static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2810 struct skl_pipe_wm *out)
2811{
2812 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2813 enum pipe pipe = crtc->pipe;
2814 int level, max_level;
2815 enum plane_id plane_id;
2816 u32 val;
2817
2818 max_level = ilk_wm_max_level(i915);
2819
2820 for_each_plane_id_on_crtc(crtc, plane_id) {
2821 struct skl_plane_wm *wm = &out->planes[plane_id];
2822
2823 for (level = 0; level <= max_level; level++) {
2824 if (plane_id != PLANE_CURSOR)
2825 val = intel_uncore_read(&i915->uncore, PLANE_WM(pipe, plane_id, level));
2826 else
2827 val = intel_uncore_read(&i915->uncore, CUR_WM(pipe, level));
2828
2829 skl_wm_level_from_reg_val(val, &wm->wm[level]);
2830 }
2831
2832 if (plane_id != PLANE_CURSOR)
2833 val = intel_uncore_read(&i915->uncore, PLANE_WM_TRANS(pipe, plane_id));
2834 else
2835 val = intel_uncore_read(&i915->uncore, CUR_WM_TRANS(pipe));
2836
2837 skl_wm_level_from_reg_val(val, &wm->trans_wm);
2838
2839 if (HAS_HW_SAGV_WM(i915)) {
2840 if (plane_id != PLANE_CURSOR)
2841 val = intel_uncore_read(&i915->uncore,
2842 PLANE_WM_SAGV(pipe, plane_id));
2843 else
2844 val = intel_uncore_read(&i915->uncore,
2845 CUR_WM_SAGV(pipe));
2846
2847 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2848
2849 if (plane_id != PLANE_CURSOR)
2850 val = intel_uncore_read(&i915->uncore,
2851 PLANE_WM_SAGV_TRANS(pipe, plane_id));
2852 else
2853 val = intel_uncore_read(&i915->uncore,
2854 CUR_WM_SAGV_TRANS(pipe));
2855
2856 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2857 } else if (DISPLAY_VER(i915) >= 12) {
2858 wm->sagv.wm0 = wm->wm[0];
2859 wm->sagv.trans_wm = wm->trans_wm;
2860 }
2861 }
2862}
2863
2864void skl_wm_get_hw_state(struct drm_i915_private *i915)
2865{
2866 struct intel_dbuf_state *dbuf_state =
2867 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2868 struct intel_crtc *crtc;
2869
2870 if (HAS_MBUS_JOINING(i915))
2871 dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
2872
2873 for_each_intel_crtc(&i915->drm, crtc) {
2874 struct intel_crtc_state *crtc_state =
2875 to_intel_crtc_state(crtc->base.state);
2876 enum pipe pipe = crtc->pipe;
2877 unsigned int mbus_offset;
2878 enum plane_id plane_id;
2879 u8 slices;
2880
2881 memset(&crtc_state->wm.skl.optimal, 0,
2882 sizeof(crtc_state->wm.skl.optimal));
2883 if (crtc_state->hw.active)
2884 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
2885 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
2886
2887 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
2888
2889 for_each_plane_id_on_crtc(crtc, plane_id) {
2890 struct skl_ddb_entry *ddb =
2891 &crtc_state->wm.skl.plane_ddb[plane_id];
2892 struct skl_ddb_entry *ddb_y =
2893 &crtc_state->wm.skl.plane_ddb_y[plane_id];
2894
2895 if (!crtc_state->hw.active)
2896 continue;
2897
2898 skl_ddb_get_hw_plane_state(i915, crtc->pipe,
2899 plane_id, ddb, ddb_y);
2900
2901 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
2902 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
2903 }
2904
2905 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
2906
2907 /*
2908 * Used for checking overlaps, so we need absolute
2909 * offsets instead of MBUS relative offsets.
2910 */
2911 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
2912 dbuf_state->joined_mbus);
2913 mbus_offset = mbus_ddb_offset(i915, slices);
2914 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
2915 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
2916
2917 /* The slices actually used by the planes on the pipe */
2918 dbuf_state->slices[pipe] =
2919 skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
2920
2921 drm_dbg_kms(&i915->drm,
2922 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
2923 crtc->base.base.id, crtc->base.name,
2924 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
2925 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
2926 str_yes_no(dbuf_state->joined_mbus));
2927 }
2928
2929 dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
2930}
2931
2932static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
2933{
2934 const struct intel_dbuf_state *dbuf_state =
2935 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2936 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
2937 struct intel_crtc *crtc;
2938
2939 for_each_intel_crtc(&i915->drm, crtc) {
2940 const struct intel_crtc_state *crtc_state =
2941 to_intel_crtc_state(crtc->base.state);
2942
2943 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
2944 }
2945
2946 for_each_intel_crtc(&i915->drm, crtc) {
2947 const struct intel_crtc_state *crtc_state =
2948 to_intel_crtc_state(crtc->base.state);
2949 u8 slices;
2950
2951 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
2952 dbuf_state->joined_mbus);
2953 if (dbuf_state->slices[crtc->pipe] & ~slices)
2954 return true;
2955
2956 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
2957 I915_MAX_PIPES, crtc->pipe))
2958 return true;
2959 }
2960
2961 return false;
2962}
2963
2964void skl_wm_sanitize(struct drm_i915_private *i915)
2965{
2966 struct intel_crtc *crtc;
2967
2968 /*
2969 * On TGL/RKL (at least) the BIOS likes to assign the planes
2970 * to the wrong DBUF slices. This will cause an infinite loop
2971 * in skl_commit_modeset_enables() as it can't find a way to
2972 * transition between the old bogus DBUF layout to the new
2973 * proper DBUF layout without DBUF allocation overlaps between
2974 * the planes (which cannot be allowed or else the hardware
2975 * may hang). If we detect a bogus DBUF layout just turn off
2976 * all the planes so that skl_commit_modeset_enables() can
2977 * simply ignore them.
2978 */
2979 if (!skl_dbuf_is_misconfigured(i915))
2980 return;
2981
2982 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
2983
2984 for_each_intel_crtc(&i915->drm, crtc) {
2985 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2986 const struct intel_plane_state *plane_state =
2987 to_intel_plane_state(plane->base.state);
2988 struct intel_crtc_state *crtc_state =
2989 to_intel_crtc_state(crtc->base.state);
2990
2991 if (plane_state->uapi.visible)
2992 intel_plane_disable_noatomic(crtc, plane);
2993
2994 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
2995
2996 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
2997 }
2998}
2999
3000void intel_wm_state_verify(struct intel_crtc *crtc,
3001 struct intel_crtc_state *new_crtc_state)
3002{
3003 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3004 struct skl_hw_state {
3005 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3006 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3007 struct skl_pipe_wm wm;
3008 } *hw;
3009 const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3010 int level, max_level = ilk_wm_max_level(i915);
3011 struct intel_plane *plane;
3012 u8 hw_enabled_slices;
3013
3014 if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3015 return;
3016
3017 hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3018 if (!hw)
3019 return;
3020
3021 skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3022
3023 skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3024
3025 hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3026
3027 if (DISPLAY_VER(i915) >= 11 &&
3028 hw_enabled_slices != i915->display.dbuf.enabled_slices)
3029 drm_err(&i915->drm,
3030 "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3031 i915->display.dbuf.enabled_slices,
3032 hw_enabled_slices);
3033
3034 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3035 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3036 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3037
3038 /* Watermarks */
3039 for (level = 0; level <= max_level; level++) {
3040 hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3041 sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3042
3043 if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3044 continue;
3045
3046 drm_err(&i915->drm,
3047 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3048 plane->base.base.id, plane->base.name, level,
3049 sw_wm_level->enable,
3050 sw_wm_level->blocks,
3051 sw_wm_level->lines,
3052 hw_wm_level->enable,
3053 hw_wm_level->blocks,
3054 hw_wm_level->lines);
3055 }
3056
3057 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3058 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3059
3060 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3061 drm_err(&i915->drm,
3062 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3063 plane->base.base.id, plane->base.name,
3064 sw_wm_level->enable,
3065 sw_wm_level->blocks,
3066 sw_wm_level->lines,
3067 hw_wm_level->enable,
3068 hw_wm_level->blocks,
3069 hw_wm_level->lines);
3070 }
3071
3072 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3073 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3074
3075 if (HAS_HW_SAGV_WM(i915) &&
3076 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3077 drm_err(&i915->drm,
3078 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3079 plane->base.base.id, plane->base.name,
3080 sw_wm_level->enable,
3081 sw_wm_level->blocks,
3082 sw_wm_level->lines,
3083 hw_wm_level->enable,
3084 hw_wm_level->blocks,
3085 hw_wm_level->lines);
3086 }
3087
3088 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3089 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3090
3091 if (HAS_HW_SAGV_WM(i915) &&
3092 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3093 drm_err(&i915->drm,
3094 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3095 plane->base.base.id, plane->base.name,
3096 sw_wm_level->enable,
3097 sw_wm_level->blocks,
3098 sw_wm_level->lines,
3099 hw_wm_level->enable,
3100 hw_wm_level->blocks,
3101 hw_wm_level->lines);
3102 }
3103
3104 /* DDB */
3105 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3106 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3107
3108 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3109 drm_err(&i915->drm,
3110 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3111 plane->base.base.id, plane->base.name,
3112 sw_ddb_entry->start, sw_ddb_entry->end,
3113 hw_ddb_entry->start, hw_ddb_entry->end);
3114 }
3115 }
3116
3117 kfree(hw);
3118}
3119
3120bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3121{
3122 return i915->display.wm.ipc_enabled;
3123}
3124
3125void skl_watermark_ipc_update(struct drm_i915_private *i915)
3126{
3127 if (!HAS_IPC(i915))
3128 return;
3129
3130 intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3131 skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3132}
3133
3134static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3135{
3136 /* Display WA #0477 WaDisableIPC: skl */
3137 if (IS_SKYLAKE(i915))
3138 return false;
3139
3140 /* Display WA #1141: SKL:all KBL:all CFL */
3141 if (IS_KABYLAKE(i915) ||
3142 IS_COFFEELAKE(i915) ||
3143 IS_COMETLAKE(i915))
3144 return i915->dram_info.symmetric_memory;
3145
3146 return true;
3147}
3148
3149void skl_watermark_ipc_init(struct drm_i915_private *i915)
3150{
3151 if (!HAS_IPC(i915))
3152 return;
3153
3154 i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3155
3156 skl_watermark_ipc_update(i915);
3157}
3158
3159static void
3160adjust_wm_latency(struct drm_i915_private *i915,
3161 u16 wm[], int max_level, int read_latency)
3162{
3163 bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3164 int i, level;
3165
3166 /*
3167 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3168 * need to be disabled. We make sure to sanitize the values out
3169 * of the punit to satisfy this requirement.
3170 */
3171 for (level = 1; level <= max_level; level++) {
3172 if (wm[level] == 0) {
3173 for (i = level + 1; i <= max_level; i++)
3174 wm[i] = 0;
3175
3176 max_level = level - 1;
3177 break;
3178 }
3179 }
3180
3181 /*
3182 * WaWmMemoryReadLatency
3183 *
3184 * punit doesn't take into account the read latency so we need
3185 * to add proper adjustement to each valid level we retrieve
3186 * from the punit when level 0 response data is 0us.
3187 */
3188 if (wm[0] == 0) {
3189 for (level = 0; level <= max_level; level++)
3190 wm[level] += read_latency;
3191 }
3192
3193 /*
3194 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3195 * If we could not get dimm info enable this WA to prevent from
3196 * any underrun. If not able to get Dimm info assume 16GB dimm
3197 * to avoid any underrun.
3198 */
3199 if (wm_lv_0_adjust_needed)
3200 wm[0] += 1;
3201}
3202
3203static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3204{
3205 struct intel_uncore *uncore = &i915->uncore;
3206 int max_level = ilk_wm_max_level(i915);
3207 u32 val;
3208
3209 val = intel_uncore_read(uncore, MTL_LATENCY_LP0_LP1);
3210 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3211 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3212
3213 val = intel_uncore_read(uncore, MTL_LATENCY_LP2_LP3);
3214 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3215 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3216
3217 val = intel_uncore_read(uncore, MTL_LATENCY_LP4_LP5);
3218 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3219 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3220
3221 adjust_wm_latency(i915, wm, max_level, 6);
3222}
3223
3224static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3225{
3226 int max_level = ilk_wm_max_level(i915);
3227 int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3228 int mult = IS_DG2(i915) ? 2 : 1;
3229 u32 val;
3230 int ret;
3231
3232 /* read the first set of memory latencies[0:3] */
3233 val = 0; /* data0 to be programmed to 0 for first set */
3234 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3235 if (ret) {
3236 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3237 return;
3238 }
3239
3240 wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3241 wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3242 wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3243 wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3244
3245 /* read the second set of memory latencies[4:7] */
3246 val = 1; /* data0 to be programmed to 1 for second set */
3247 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3248 if (ret) {
3249 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3250 return;
3251 }
3252
3253 wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3254 wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3255 wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3256 wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3257
3258 adjust_wm_latency(i915, wm, max_level, read_latency);
3259}
3260
3261static void skl_setup_wm_latency(struct drm_i915_private *i915)
3262{
3263 if (DISPLAY_VER(i915) >= 14)
3264 mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3265 else
3266 skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3267
3268 intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3269}
3270
3271static const struct intel_wm_funcs skl_wm_funcs = {
3272 .compute_global_watermarks = skl_compute_wm,
3273};
3274
3275void skl_wm_init(struct drm_i915_private *i915)
3276{
3277 intel_sagv_init(i915);
3278
3279 skl_setup_wm_latency(i915);
3280
3281 i915->display.funcs.wm = &skl_wm_funcs;
3282}
3283
3284static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3285{
3286 struct intel_dbuf_state *dbuf_state;
3287
3288 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3289 if (!dbuf_state)
3290 return NULL;
3291
3292 return &dbuf_state->base;
3293}
3294
3295static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3296 struct intel_global_state *state)
3297{
3298 kfree(state);
3299}
3300
3301static const struct intel_global_state_funcs intel_dbuf_funcs = {
3302 .atomic_duplicate_state = intel_dbuf_duplicate_state,
3303 .atomic_destroy_state = intel_dbuf_destroy_state,
3304};
3305
3306struct intel_dbuf_state *
3307intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3308{
3309 struct drm_i915_private *i915 = to_i915(state->base.dev);
3310 struct intel_global_state *dbuf_state;
3311
3312 dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3313 if (IS_ERR(dbuf_state))
3314 return ERR_CAST(dbuf_state);
3315
3316 return to_intel_dbuf_state(dbuf_state);
3317}
3318
3319int intel_dbuf_init(struct drm_i915_private *i915)
3320{
3321 struct intel_dbuf_state *dbuf_state;
3322
3323 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3324 if (!dbuf_state)
3325 return -ENOMEM;
3326
3327 intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3328 &dbuf_state->base, &intel_dbuf_funcs);
3329
3330 return 0;
3331}
3332
3333/*
3334 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3335 * update the request state of all DBUS slices.
3336 */
3337static void update_mbus_pre_enable(struct intel_atomic_state *state)
3338{
3339 struct drm_i915_private *i915 = to_i915(state->base.dev);
3340 u32 mbus_ctl, dbuf_min_tracker_val;
3341 enum dbuf_slice slice;
3342 const struct intel_dbuf_state *dbuf_state =
3343 intel_atomic_get_new_dbuf_state(state);
3344
3345 if (!HAS_MBUS_JOINING(i915))
3346 return;
3347
3348 /*
3349 * TODO: Implement vblank synchronized MBUS joining changes.
3350 * Must be properly coordinated with dbuf reprogramming.
3351 */
3352 if (dbuf_state->joined_mbus) {
3353 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3354 MBUS_JOIN_PIPE_SELECT_NONE;
3355 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3356 } else {
3357 mbus_ctl = MBUS_HASHING_MODE_2x2 |
3358 MBUS_JOIN_PIPE_SELECT_NONE;
3359 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3360 }
3361
3362 intel_de_rmw(i915, MBUS_CTL,
3363 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3364 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3365
3366 for_each_dbuf_slice(i915, slice)
3367 intel_de_rmw(i915, DBUF_CTL_S(slice),
3368 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3369 dbuf_min_tracker_val);
3370}
3371
3372void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3373{
3374 struct drm_i915_private *i915 = to_i915(state->base.dev);
3375 const struct intel_dbuf_state *new_dbuf_state =
3376 intel_atomic_get_new_dbuf_state(state);
3377 const struct intel_dbuf_state *old_dbuf_state =
3378 intel_atomic_get_old_dbuf_state(state);
3379
3380 if (!new_dbuf_state ||
3381 (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3382 new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3383 return;
3384
3385 WARN_ON(!new_dbuf_state->base.changed);
3386
3387 update_mbus_pre_enable(state);
3388 gen9_dbuf_slices_update(i915,
3389 old_dbuf_state->enabled_slices |
3390 new_dbuf_state->enabled_slices);
3391}
3392
3393void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3394{
3395 struct drm_i915_private *i915 = to_i915(state->base.dev);
3396 const struct intel_dbuf_state *new_dbuf_state =
3397 intel_atomic_get_new_dbuf_state(state);
3398 const struct intel_dbuf_state *old_dbuf_state =
3399 intel_atomic_get_old_dbuf_state(state);
3400
3401 if (!new_dbuf_state ||
3402 (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3403 new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3404 return;
3405
3406 WARN_ON(!new_dbuf_state->base.changed);
3407
3408 gen9_dbuf_slices_update(i915,
3409 new_dbuf_state->enabled_slices);
3410}
3411
3412static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3413{
3414 switch (pipe) {
3415 case PIPE_A:
3416 return !(active_pipes & BIT(PIPE_D));
3417 case PIPE_D:
3418 return !(active_pipes & BIT(PIPE_A));
3419 case PIPE_B:
3420 return !(active_pipes & BIT(PIPE_C));
3421 case PIPE_C:
3422 return !(active_pipes & BIT(PIPE_B));
3423 default: /* to suppress compiler warning */
3424 MISSING_CASE(pipe);
3425 break;
3426 }
3427
3428 return false;
3429}
3430
3431void intel_mbus_dbox_update(struct intel_atomic_state *state)
3432{
3433 struct drm_i915_private *i915 = to_i915(state->base.dev);
3434 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3435 const struct intel_crtc_state *new_crtc_state;
3436 const struct intel_crtc *crtc;
3437 u32 val = 0;
3438 int i;
3439
3440 if (DISPLAY_VER(i915) < 11)
3441 return;
3442
3443 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3444 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3445 if (!new_dbuf_state ||
3446 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3447 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3448 return;
3449
3450 if (DISPLAY_VER(i915) >= 14)
3451 val |= MBUS_DBOX_I_CREDIT(2);
3452
3453 if (DISPLAY_VER(i915) >= 12) {
3454 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3455 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3456 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3457 }
3458
3459 if (DISPLAY_VER(i915) >= 14)
3460 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3461 MBUS_DBOX_A_CREDIT(8);
3462 else if (IS_ALDERLAKE_P(i915))
3463 /* Wa_22010947358:adl-p */
3464 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3465 MBUS_DBOX_A_CREDIT(4);
3466 else
3467 val |= MBUS_DBOX_A_CREDIT(2);
3468
3469 if (DISPLAY_VER(i915) >= 14) {
3470 val |= MBUS_DBOX_B_CREDIT(0xA);
3471 } else if (IS_ALDERLAKE_P(i915)) {
3472 val |= MBUS_DBOX_BW_CREDIT(2);
3473 val |= MBUS_DBOX_B_CREDIT(8);
3474 } else if (DISPLAY_VER(i915) >= 12) {
3475 val |= MBUS_DBOX_BW_CREDIT(2);
3476 val |= MBUS_DBOX_B_CREDIT(12);
3477 } else {
3478 val |= MBUS_DBOX_BW_CREDIT(1);
3479 val |= MBUS_DBOX_B_CREDIT(8);
3480 }
3481
3482 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3483 u32 pipe_val = val;
3484
3485 if (!new_crtc_state->hw.active)
3486 continue;
3487
3488 if (DISPLAY_VER(i915) >= 14) {
3489 if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3490 new_dbuf_state->active_pipes))
3491 pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3492 else
3493 pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3494 }
3495
3496 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3497 }
3498}
3499
3500static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3501{
3502 struct drm_i915_private *i915 = m->private;
3503
3504 seq_printf(m, "Isochronous Priority Control: %s\n",
3505 str_yes_no(skl_watermark_ipc_enabled(i915)));
3506 return 0;
3507}
3508
3509static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3510{
3511 struct drm_i915_private *i915 = inode->i_private;
3512
3513 return single_open(file, skl_watermark_ipc_status_show, i915);
3514}
3515
3516static ssize_t skl_watermark_ipc_status_write(struct file *file,
3517 const char __user *ubuf,
3518 size_t len, loff_t *offp)
3519{
3520 struct seq_file *m = file->private_data;
3521 struct drm_i915_private *i915 = m->private;
3522 intel_wakeref_t wakeref;
3523 bool enable;
3524 int ret;
3525
3526 ret = kstrtobool_from_user(ubuf, len, &enable);
3527 if (ret < 0)
3528 return ret;
3529
3530 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3531 if (!skl_watermark_ipc_enabled(i915) && enable)
3532 drm_info(&i915->drm,
3533 "Enabling IPC: WM will be proper only after next commit\n");
3534 i915->display.wm.ipc_enabled = enable;
3535 skl_watermark_ipc_update(i915);
3536 }
3537
3538 return len;
3539}
3540
3541static const struct file_operations skl_watermark_ipc_status_fops = {
3542 .owner = THIS_MODULE,
3543 .open = skl_watermark_ipc_status_open,
3544 .read = seq_read,
3545 .llseek = seq_lseek,
3546 .release = single_release,
3547 .write = skl_watermark_ipc_status_write
3548};
3549
3550void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915)
3551{
3552 struct drm_minor *minor = i915->drm.primary;
3553
3554 if (!HAS_IPC(i915))
3555 return;
3556
3557 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3558 &skl_watermark_ipc_status_fops);
3559}