1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2022 Intel Corporation
   4 */
   5
   6#include <drm/drm_blend.h>
   7
   8#include "i915_drv.h"
   9#include "i915_fixed.h"
  10#include "i915_reg.h"
  11#include "i9xx_wm.h"
  12#include "intel_atomic.h"
  13#include "intel_atomic_plane.h"
  14#include "intel_bw.h"
  15#include "intel_crtc.h"
  16#include "intel_de.h"
  17#include "intel_display.h"
  18#include "intel_display_power.h"
  19#include "intel_display_types.h"
  20#include "intel_fb.h"
  21#include "intel_pcode.h"
  22#include "intel_wm.h"
  23#include "skl_watermark.h"
  24#include "skl_watermark_regs.h"
  25
  26static void skl_sagv_disable(struct drm_i915_private *i915);
  27
  28/* Stores plane specific WM parameters */
  29struct skl_wm_params {
  30	bool x_tiled, y_tiled;
  31	bool rc_surface;
  32	bool is_planar;
  33	u32 width;
  34	u8 cpp;
  35	u32 plane_pixel_rate;
  36	u32 y_min_scanlines;
  37	u32 plane_bytes_per_line;
  38	uint_fixed_16_16_t plane_blocks_per_line;
  39	uint_fixed_16_16_t y_tile_minimum;
  40	u32 linetime_us;
  41	u32 dbuf_block_size;
  42};
  43
  44u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
  45{
  46	u8 enabled_slices = 0;
  47	enum dbuf_slice slice;
  48
  49	for_each_dbuf_slice(i915, slice) {
  50		if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
  51			enabled_slices |= BIT(slice);
  52	}
  53
  54	return enabled_slices;
  55}
  56
  57/*
  58 * FIXME: We still don't have the proper code detect if we need to apply the WA,
  59 * so assume we'll always need it in order to avoid underruns.
  60 */
  61static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
  62{
  63	return DISPLAY_VER(i915) == 9;
  64}
  65
  66static bool
  67intel_has_sagv(struct drm_i915_private *i915)
  68{
  69	return HAS_SAGV(i915) &&
  70		i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
  71}
  72
  73static u32
  74intel_sagv_block_time(struct drm_i915_private *i915)
  75{
  76	if (DISPLAY_VER(i915) >= 14) {
  77		u32 val;
  78
  79		val = intel_de_read(i915, MTL_LATENCY_SAGV);
  80
  81		return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
  82	} else if (DISPLAY_VER(i915) >= 12) {
  83		u32 val = 0;
  84		int ret;
  85
  86		ret = snb_pcode_read(&i915->uncore,
  87				     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
  88				     &val, NULL);
  89		if (ret) {
  90			drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
  91			return 0;
  92		}
  93
  94		return val;
  95	} else if (DISPLAY_VER(i915) == 11) {
  96		return 10;
  97	} else if (HAS_SAGV(i915)) {
  98		return 30;
  99	} else {
 100		return 0;
 101	}
 102}
 103
 104static void intel_sagv_init(struct drm_i915_private *i915)
 105{
 106	if (!HAS_SAGV(i915))
 107		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 108
 109	/*
 110	 * Probe to see if we have working SAGV control.
 111	 * For icl+ this was already determined by intel_bw_init_hw().
 112	 */
 113	if (DISPLAY_VER(i915) < 11)
 114		skl_sagv_disable(i915);
 115
 116	drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
 117
 118	i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
 119
 120	drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
 121		    str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
 122
 123	/* avoid overflow when adding with wm0 latency/etc. */
 124	if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
 125		     "Excessive SAGV block time %u, ignoring\n",
 126		     i915->display.sagv.block_time_us))
 127		i915->display.sagv.block_time_us = 0;
 128
 129	if (!intel_has_sagv(i915))
 130		i915->display.sagv.block_time_us = 0;
 131}
 132
 133/*
 134 * SAGV dynamically adjusts the system agent voltage and clock frequencies
 135 * depending on power and performance requirements. The display engine access
 136 * to system memory is blocked during the adjustment time. Because of the
 137 * blocking time, having this enabled can cause full system hangs and/or pipe
 138 * underruns if we don't meet all of the following requirements:
 139 *
 140 *  - <= 1 pipe enabled
 141 *  - All planes can enable watermarks for latencies >= SAGV engine block time
 142 *  - We're not using an interlaced display configuration
 143 */
 144static void skl_sagv_enable(struct drm_i915_private *i915)
 145{
 146	int ret;
 147
 148	if (!intel_has_sagv(i915))
 149		return;
 150
 151	if (i915->display.sagv.status == I915_SAGV_ENABLED)
 152		return;
 153
 154	drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
 155	ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 156			      GEN9_SAGV_ENABLE);
 157
 158	/* We don't need to wait for SAGV when enabling */
 159
 160	/*
 161	 * Some skl systems, pre-release machines in particular,
 162	 * don't actually have SAGV.
 163	 */
 164	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 165		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 166		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 167		return;
 168	} else if (ret < 0) {
 169		drm_err(&i915->drm, "Failed to enable SAGV\n");
 170		return;
 171	}
 172
 173	i915->display.sagv.status = I915_SAGV_ENABLED;
 174}
 175
 176static void skl_sagv_disable(struct drm_i915_private *i915)
 177{
 178	int ret;
 179
 180	if (!intel_has_sagv(i915))
 181		return;
 182
 183	if (i915->display.sagv.status == I915_SAGV_DISABLED)
 184		return;
 185
 186	drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
 187	/* bspec says to keep retrying for at least 1 ms */
 188	ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 189				GEN9_SAGV_DISABLE,
 190				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
 191				1);
 192	/*
 193	 * Some skl systems, pre-release machines in particular,
 194	 * don't actually have SAGV.
 195	 */
 196	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 197		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 198		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 199		return;
 200	} else if (ret < 0) {
 201		drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
 202		return;
 203	}
 204
 205	i915->display.sagv.status = I915_SAGV_DISABLED;
 206}
 207
 208static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
 209{
 210	struct drm_i915_private *i915 = to_i915(state->base.dev);
 211	const struct intel_bw_state *new_bw_state =
 212		intel_atomic_get_new_bw_state(state);
 213
 214	if (!new_bw_state)
 215		return;
 216
 217	if (!intel_can_enable_sagv(i915, new_bw_state))
 218		skl_sagv_disable(i915);
 219}
 220
 221static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
 222{
 223	struct drm_i915_private *i915 = to_i915(state->base.dev);
 224	const struct intel_bw_state *new_bw_state =
 225		intel_atomic_get_new_bw_state(state);
 226
 227	if (!new_bw_state)
 228		return;
 229
 230	if (intel_can_enable_sagv(i915, new_bw_state))
 231		skl_sagv_enable(i915);
 232}
 233
 234static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
 235{
 236	struct drm_i915_private *i915 = to_i915(state->base.dev);
 237	const struct intel_bw_state *old_bw_state =
 238		intel_atomic_get_old_bw_state(state);
 239	const struct intel_bw_state *new_bw_state =
 240		intel_atomic_get_new_bw_state(state);
 241	u16 old_mask, new_mask;
 242
 243	if (!new_bw_state)
 244		return;
 245
 246	old_mask = old_bw_state->qgv_points_mask;
 247	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 248
 249	if (old_mask == new_mask)
 250		return;
 251
 252	WARN_ON(!new_bw_state->base.changed);
 253
 254	drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
 255		    old_mask, new_mask);
 256
 257	/*
 258	 * Restrict required qgv points before updating the configuration.
 259	 * According to BSpec we can't mask and unmask qgv points at the same
 260	 * time. Also masking should be done before updating the configuration
 261	 * and unmasking afterwards.
 262	 */
 263	icl_pcode_restrict_qgv_points(i915, new_mask);
 264}
 265
 266static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
 267{
 268	struct drm_i915_private *i915 = to_i915(state->base.dev);
 269	const struct intel_bw_state *old_bw_state =
 270		intel_atomic_get_old_bw_state(state);
 271	const struct intel_bw_state *new_bw_state =
 272		intel_atomic_get_new_bw_state(state);
 273	u16 old_mask, new_mask;
 274
 275	if (!new_bw_state)
 276		return;
 277
 278	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 279	new_mask = new_bw_state->qgv_points_mask;
 280
 281	if (old_mask == new_mask)
 282		return;
 283
 284	WARN_ON(!new_bw_state->base.changed);
 285
 286	drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
 287		    old_mask, new_mask);
 288
 289	/*
 290	 * Allow required qgv points after updating the configuration.
 291	 * According to BSpec we can't mask and unmask qgv points at the same
 292	 * time. Also masking should be done before updating the configuration
 293	 * and unmasking afterwards.
 294	 */
 295	icl_pcode_restrict_qgv_points(i915, new_mask);
 296}
 297
 298void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
 299{
 300	struct drm_i915_private *i915 = to_i915(state->base.dev);
 301
 302	/*
 303	 * Just return if we can't control SAGV or don't have it.
 304	 * This is different from situation when we have SAGV but just can't
 305	 * afford it due to DBuf limitation - in case if SAGV is completely
 306	 * disabled in a BIOS, we are not even allowed to send a PCode request,
 307	 * as it will throw an error. So have to check it here.
 308	 */
 309	if (!intel_has_sagv(i915))
 310		return;
 311
 312	if (DISPLAY_VER(i915) >= 11)
 313		icl_sagv_pre_plane_update(state);
 314	else
 315		skl_sagv_pre_plane_update(state);
 316}
 317
 318void intel_sagv_post_plane_update(struct intel_atomic_state *state)
 319{
 320	struct drm_i915_private *i915 = to_i915(state->base.dev);
 321
 322	/*
 323	 * Just return if we can't control SAGV or don't have it.
 324	 * This is different from situation when we have SAGV but just can't
 325	 * afford it due to DBuf limitation - in case if SAGV is completely
 326	 * disabled in a BIOS, we are not even allowed to send a PCode request,
 327	 * as it will throw an error. So have to check it here.
 328	 */
 329	if (!intel_has_sagv(i915))
 330		return;
 331
 332	if (DISPLAY_VER(i915) >= 11)
 333		icl_sagv_post_plane_update(state);
 334	else
 335		skl_sagv_post_plane_update(state);
 336}
 337
 338static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 339{
 340	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 341	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 342	enum plane_id plane_id;
 343	int max_level = INT_MAX;
 344
 345	if (!intel_has_sagv(i915))
 346		return false;
 347
 348	if (!crtc_state->hw.active)
 349		return true;
 350
 351	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
 352		return false;
 353
 354	for_each_plane_id_on_crtc(crtc, plane_id) {
 355		const struct skl_plane_wm *wm =
 356			&crtc_state->wm.skl.optimal.planes[plane_id];
 357		int level;
 358
 359		/* Skip this plane if it's not enabled */
 360		if (!wm->wm[0].enable)
 361			continue;
 362
 363		/* Find the highest enabled wm level for this plane */
 364		for (level = i915->display.wm.num_levels - 1;
 365		     !wm->wm[level].enable; --level)
 366		     { }
 367
 368		/* Highest common enabled wm level for all planes */
 369		max_level = min(level, max_level);
 370	}
 371
 372	/* No enabled planes? */
 373	if (max_level == INT_MAX)
 374		return true;
 375
 376	for_each_plane_id_on_crtc(crtc, plane_id) {
 377		const struct skl_plane_wm *wm =
 378			&crtc_state->wm.skl.optimal.planes[plane_id];
 379
 380		/*
 381		 * All enabled planes must have enabled a common wm level that
 382		 * can tolerate memory latencies higher than sagv_block_time_us
 383		 */
 384		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
 385			return false;
 386	}
 387
 388	return true;
 389}
 390
 391static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 392{
 393	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 394	enum plane_id plane_id;
 395
 396	if (!crtc_state->hw.active)
 397		return true;
 398
 399	for_each_plane_id_on_crtc(crtc, plane_id) {
 400		const struct skl_plane_wm *wm =
 401			&crtc_state->wm.skl.optimal.planes[plane_id];
 402
 403		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
 404			return false;
 405	}
 406
 407	return true;
 408}
 409
 410static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 411{
 412	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 413	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 414
 415	if (!i915->display.params.enable_sagv)
 416		return false;
 417
 418	if (DISPLAY_VER(i915) >= 12)
 419		return tgl_crtc_can_enable_sagv(crtc_state);
 420	else
 421		return skl_crtc_can_enable_sagv(crtc_state);
 422}
 423
 424bool intel_can_enable_sagv(struct drm_i915_private *i915,
 425			   const struct intel_bw_state *bw_state)
 426{
 427	if (DISPLAY_VER(i915) < 11 &&
 428	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
 429		return false;
 430
 431	return bw_state->pipe_sagv_reject == 0;
 432}
 433
 434static int intel_compute_sagv_mask(struct intel_atomic_state *state)
 435{
 436	struct drm_i915_private *i915 = to_i915(state->base.dev);
 437	int ret;
 438	struct intel_crtc *crtc;
 439	struct intel_crtc_state *new_crtc_state;
 440	struct intel_bw_state *new_bw_state = NULL;
 441	const struct intel_bw_state *old_bw_state = NULL;
 442	int i;
 443
 444	for_each_new_intel_crtc_in_state(state, crtc,
 445					 new_crtc_state, i) {
 446		new_bw_state = intel_atomic_get_bw_state(state);
 447		if (IS_ERR(new_bw_state))
 448			return PTR_ERR(new_bw_state);
 449
 450		old_bw_state = intel_atomic_get_old_bw_state(state);
 451
 452		if (intel_crtc_can_enable_sagv(new_crtc_state))
 453			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
 454		else
 455			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
 456	}
 457
 458	if (!new_bw_state)
 459		return 0;
 460
 461	new_bw_state->active_pipes =
 462		intel_calc_active_pipes(state, old_bw_state->active_pipes);
 463
 464	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
 465		ret = intel_atomic_lock_global_state(&new_bw_state->base);
 466		if (ret)
 467			return ret;
 468	}
 469
 470	if (intel_can_enable_sagv(i915, new_bw_state) !=
 471	    intel_can_enable_sagv(i915, old_bw_state)) {
 472		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 473		if (ret)
 474			return ret;
 475	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
 476		ret = intel_atomic_lock_global_state(&new_bw_state->base);
 477		if (ret)
 478			return ret;
 479	}
 480
 481	for_each_new_intel_crtc_in_state(state, crtc,
 482					 new_crtc_state, i) {
 483		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
 484
 485		/*
 486		 * We store use_sagv_wm in the crtc state rather than relying on
 487		 * that bw state since we have no convenient way to get at the
 488		 * latter from the plane commit hooks (especially in the legacy
 489		 * cursor case)
 490		 */
 491		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
 492			DISPLAY_VER(i915) >= 12 &&
 493			intel_can_enable_sagv(i915, new_bw_state);
 494	}
 495
 496	return 0;
 497}
 498
 499static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
 500			      u16 start, u16 end)
 501{
 502	entry->start = start;
 503	entry->end = end;
 504
 505	return end;
 506}
 507
 508static int intel_dbuf_slice_size(struct drm_i915_private *i915)
 509{
 510	return DISPLAY_INFO(i915)->dbuf.size /
 511		hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
 512}
 513
 514static void
 515skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
 516			 struct skl_ddb_entry *ddb)
 517{
 518	int slice_size = intel_dbuf_slice_size(i915);
 519
 520	if (!slice_mask) {
 521		ddb->start = 0;
 522		ddb->end = 0;
 523		return;
 524	}
 525
 526	ddb->start = (ffs(slice_mask) - 1) * slice_size;
 527	ddb->end = fls(slice_mask) * slice_size;
 528
 529	WARN_ON(ddb->start >= ddb->end);
 530	WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
 531}
 532
 533static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
 534{
 535	struct skl_ddb_entry ddb;
 536
 537	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
 538		slice_mask = BIT(DBUF_S1);
 539	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
 540		slice_mask = BIT(DBUF_S3);
 541
 542	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
 543
 544	return ddb.start;
 545}
 546
 547u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
 548			    const struct skl_ddb_entry *entry)
 549{
 550	int slice_size = intel_dbuf_slice_size(i915);
 551	enum dbuf_slice start_slice, end_slice;
 552	u8 slice_mask = 0;
 553
 554	if (!skl_ddb_entry_size(entry))
 555		return 0;
 556
 557	start_slice = entry->start / slice_size;
 558	end_slice = (entry->end - 1) / slice_size;
 559
 560	/*
 561	 * Per plane DDB entry can in a really worst case be on multiple slices
 562	 * but single entry is anyway contigious.
 563	 */
 564	while (start_slice <= end_slice) {
 565		slice_mask |= BIT(start_slice);
 566		start_slice++;
 567	}
 568
 569	return slice_mask;
 570}
 571
 572static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
 573{
 574	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
 575	int hdisplay, vdisplay;
 576
 577	if (!crtc_state->hw.active)
 578		return 0;
 579
 580	/*
 581	 * Watermark/ddb requirement highly depends upon width of the
 582	 * framebuffer, So instead of allocating DDB equally among pipes
 583	 * distribute DDB based on resolution/width of the display.
 584	 */
 585	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
 586
 587	return hdisplay;
 588}
 589
 590static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
 591				    enum pipe for_pipe,
 592				    unsigned int *weight_start,
 593				    unsigned int *weight_end,
 594				    unsigned int *weight_total)
 595{
 596	struct drm_i915_private *i915 =
 597		to_i915(dbuf_state->base.state->base.dev);
 598	enum pipe pipe;
 599
 600	*weight_start = 0;
 601	*weight_end = 0;
 602	*weight_total = 0;
 603
 604	for_each_pipe(i915, pipe) {
 605		int weight = dbuf_state->weight[pipe];
 606
 607		/*
 608		 * Do not account pipes using other slice sets
 609		 * luckily as of current BSpec slice sets do not partially
 610		 * intersect(pipes share either same one slice or same slice set
 611		 * i.e no partial intersection), so it is enough to check for
 612		 * equality for now.
 613		 */
 614		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
 615			continue;
 616
 617		*weight_total += weight;
 618		if (pipe < for_pipe) {
 619			*weight_start += weight;
 620			*weight_end += weight;
 621		} else if (pipe == for_pipe) {
 622			*weight_end += weight;
 623		}
 624	}
 625}
 626
 627static int
 628skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
 629{
 630	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 631	unsigned int weight_total, weight_start, weight_end;
 632	const struct intel_dbuf_state *old_dbuf_state =
 633		intel_atomic_get_old_dbuf_state(state);
 634	struct intel_dbuf_state *new_dbuf_state =
 635		intel_atomic_get_new_dbuf_state(state);
 636	struct intel_crtc_state *crtc_state;
 637	struct skl_ddb_entry ddb_slices;
 638	enum pipe pipe = crtc->pipe;
 639	unsigned int mbus_offset = 0;
 640	u32 ddb_range_size;
 641	u32 dbuf_slice_mask;
 642	u32 start, end;
 643	int ret;
 644
 645	if (new_dbuf_state->weight[pipe] == 0) {
 646		skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
 647		goto out;
 648	}
 649
 650	dbuf_slice_mask = new_dbuf_state->slices[pipe];
 651
 652	skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
 653	mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
 654	ddb_range_size = skl_ddb_entry_size(&ddb_slices);
 655
 656	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
 657				&weight_start, &weight_end, &weight_total);
 658
 659	start = ddb_range_size * weight_start / weight_total;
 660	end = ddb_range_size * weight_end / weight_total;
 661
 662	skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
 663			   ddb_slices.start - mbus_offset + start,
 664			   ddb_slices.start - mbus_offset + end);
 665
 666out:
 667	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
 668	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
 669				&new_dbuf_state->ddb[pipe]))
 670		return 0;
 671
 672	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
 673	if (ret)
 674		return ret;
 675
 676	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
 677	if (IS_ERR(crtc_state))
 678		return PTR_ERR(crtc_state);
 679
 680	/*
 681	 * Used for checking overlaps, so we need absolute
 682	 * offsets instead of MBUS relative offsets.
 683	 */
 684	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
 685	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
 686
 687	drm_dbg_kms(&i915->drm,
 688		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
 689		    crtc->base.base.id, crtc->base.name,
 690		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
 691		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
 692		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
 693		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
 694
 695	return 0;
 696}
 697
 698static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
 699				 int width, const struct drm_format_info *format,
 700				 u64 modifier, unsigned int rotation,
 701				 u32 plane_pixel_rate, struct skl_wm_params *wp,
 702				 int color_plane);
 703
 704static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 705				 struct intel_plane *plane,
 706				 int level,
 707				 unsigned int latency,
 708				 const struct skl_wm_params *wp,
 709				 const struct skl_wm_level *result_prev,
 710				 struct skl_wm_level *result /* out */);
 711
 712static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
 713				   const struct skl_wm_params *wp)
 714{
 715	unsigned int latency = i915->display.wm.skl_latency[level];
 716
 717	if (latency == 0)
 718		return 0;
 719
 720	/*
 721	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
 722	 * Display WA #1141: kbl,cfl
 723	 */
 724	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
 725	    skl_watermark_ipc_enabled(i915))
 726		latency += 4;
 727
 728	if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
 729		latency += 15;
 730
 731	return latency;
 732}
 733
 734static unsigned int
 735skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 736		      int num_active)
 737{
 738	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
 739	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 740	struct skl_wm_level wm = {};
 741	int ret, min_ddb_alloc = 0;
 742	struct skl_wm_params wp;
 743	int level;
 744
 745	ret = skl_compute_wm_params(crtc_state, 256,
 746				    drm_format_info(DRM_FORMAT_ARGB8888),
 747				    DRM_FORMAT_MOD_LINEAR,
 748				    DRM_MODE_ROTATE_0,
 749				    crtc_state->pixel_rate, &wp, 0);
 750	drm_WARN_ON(&i915->drm, ret);
 751
 752	for (level = 0; level < i915->display.wm.num_levels; level++) {
 753		unsigned int latency = skl_wm_latency(i915, level, &wp);
 754
 755		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
 756		if (wm.min_ddb_alloc == U16_MAX)
 757			break;
 758
 759		min_ddb_alloc = wm.min_ddb_alloc;
 760	}
 761
 762	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
 763}
 764
 765static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
 766{
 767	skl_ddb_entry_init(entry,
 768			   REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
 769			   REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
 770	if (entry->end)
 771		entry->end++;
 772}
 773
 774static void
 775skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
 776			   const enum pipe pipe,
 777			   const enum plane_id plane_id,
 778			   struct skl_ddb_entry *ddb,
 779			   struct skl_ddb_entry *ddb_y)
 780{
 781	u32 val;
 782
 783	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
 784	if (plane_id == PLANE_CURSOR) {
 785		val = intel_de_read(i915, CUR_BUF_CFG(pipe));
 786		skl_ddb_entry_init_from_hw(ddb, val);
 787		return;
 788	}
 789
 790	val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
 791	skl_ddb_entry_init_from_hw(ddb, val);
 792
 793	if (DISPLAY_VER(i915) >= 11)
 794		return;
 795
 796	val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
 797	skl_ddb_entry_init_from_hw(ddb_y, val);
 798}
 799
 800static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
 801				      struct skl_ddb_entry *ddb,
 802				      struct skl_ddb_entry *ddb_y)
 803{
 804	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 805	enum intel_display_power_domain power_domain;
 806	enum pipe pipe = crtc->pipe;
 807	intel_wakeref_t wakeref;
 808	enum plane_id plane_id;
 809
 810	power_domain = POWER_DOMAIN_PIPE(pipe);
 811	wakeref = intel_display_power_get_if_enabled(i915, power_domain);
 812	if (!wakeref)
 813		return;
 814
 815	for_each_plane_id_on_crtc(crtc, plane_id)
 816		skl_ddb_get_hw_plane_state(i915, pipe,
 817					   plane_id,
 818					   &ddb[plane_id],
 819					   &ddb_y[plane_id]);
 820
 821	intel_display_power_put(i915, power_domain, wakeref);
 822}
 823
 824struct dbuf_slice_conf_entry {
 825	u8 active_pipes;
 826	u8 dbuf_mask[I915_MAX_PIPES];
 827	bool join_mbus;
 828};
 829
 830/*
 831 * Table taken from Bspec 12716
 832 * Pipes do have some preferred DBuf slice affinity,
 833 * plus there are some hardcoded requirements on how
 834 * those should be distributed for multipipe scenarios.
 835 * For more DBuf slices algorithm can get even more messy
 836 * and less readable, so decided to use a table almost
 837 * as is from BSpec itself - that way it is at least easier
 838 * to compare, change and check.
 839 */
 840static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
 841/* Autogenerated with igt/tools/intel_dbuf_map tool: */
 842{
 843	{
 844		.active_pipes = BIT(PIPE_A),
 845		.dbuf_mask = {
 846			[PIPE_A] = BIT(DBUF_S1),
 847		},
 848	},
 849	{
 850		.active_pipes = BIT(PIPE_B),
 851		.dbuf_mask = {
 852			[PIPE_B] = BIT(DBUF_S1),
 853		},
 854	},
 855	{
 856		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 857		.dbuf_mask = {
 858			[PIPE_A] = BIT(DBUF_S1),
 859			[PIPE_B] = BIT(DBUF_S2),
 860		},
 861	},
 862	{
 863		.active_pipes = BIT(PIPE_C),
 864		.dbuf_mask = {
 865			[PIPE_C] = BIT(DBUF_S2),
 866		},
 867	},
 868	{
 869		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 870		.dbuf_mask = {
 871			[PIPE_A] = BIT(DBUF_S1),
 872			[PIPE_C] = BIT(DBUF_S2),
 873		},
 874	},
 875	{
 876		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 877		.dbuf_mask = {
 878			[PIPE_B] = BIT(DBUF_S1),
 879			[PIPE_C] = BIT(DBUF_S2),
 880		},
 881	},
 882	{
 883		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 884		.dbuf_mask = {
 885			[PIPE_A] = BIT(DBUF_S1),
 886			[PIPE_B] = BIT(DBUF_S1),
 887			[PIPE_C] = BIT(DBUF_S2),
 888		},
 889	},
 890	{}
 891};
 892
 893/*
 894 * Table taken from Bspec 49255
 895 * Pipes do have some preferred DBuf slice affinity,
 896 * plus there are some hardcoded requirements on how
 897 * those should be distributed for multipipe scenarios.
 898 * For more DBuf slices algorithm can get even more messy
 899 * and less readable, so decided to use a table almost
 900 * as is from BSpec itself - that way it is at least easier
 901 * to compare, change and check.
 902 */
 903static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
 904/* Autogenerated with igt/tools/intel_dbuf_map tool: */
 905{
 906	{
 907		.active_pipes = BIT(PIPE_A),
 908		.dbuf_mask = {
 909			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
 910		},
 911	},
 912	{
 913		.active_pipes = BIT(PIPE_B),
 914		.dbuf_mask = {
 915			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
 916		},
 917	},
 918	{
 919		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 920		.dbuf_mask = {
 921			[PIPE_A] = BIT(DBUF_S2),
 922			[PIPE_B] = BIT(DBUF_S1),
 923		},
 924	},
 925	{
 926		.active_pipes = BIT(PIPE_C),
 927		.dbuf_mask = {
 928			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
 929		},
 930	},
 931	{
 932		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 933		.dbuf_mask = {
 934			[PIPE_A] = BIT(DBUF_S1),
 935			[PIPE_C] = BIT(DBUF_S2),
 936		},
 937	},
 938	{
 939		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 940		.dbuf_mask = {
 941			[PIPE_B] = BIT(DBUF_S1),
 942			[PIPE_C] = BIT(DBUF_S2),
 943		},
 944	},
 945	{
 946		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 947		.dbuf_mask = {
 948			[PIPE_A] = BIT(DBUF_S1),
 949			[PIPE_B] = BIT(DBUF_S1),
 950			[PIPE_C] = BIT(DBUF_S2),
 951		},
 952	},
 953	{
 954		.active_pipes = BIT(PIPE_D),
 955		.dbuf_mask = {
 956			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
 957		},
 958	},
 959	{
 960		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
 961		.dbuf_mask = {
 962			[PIPE_A] = BIT(DBUF_S1),
 963			[PIPE_D] = BIT(DBUF_S2),
 964		},
 965	},
 966	{
 967		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
 968		.dbuf_mask = {
 969			[PIPE_B] = BIT(DBUF_S1),
 970			[PIPE_D] = BIT(DBUF_S2),
 971		},
 972	},
 973	{
 974		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
 975		.dbuf_mask = {
 976			[PIPE_A] = BIT(DBUF_S1),
 977			[PIPE_B] = BIT(DBUF_S1),
 978			[PIPE_D] = BIT(DBUF_S2),
 979		},
 980	},
 981	{
 982		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
 983		.dbuf_mask = {
 984			[PIPE_C] = BIT(DBUF_S1),
 985			[PIPE_D] = BIT(DBUF_S2),
 986		},
 987	},
 988	{
 989		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
 990		.dbuf_mask = {
 991			[PIPE_A] = BIT(DBUF_S1),
 992			[PIPE_C] = BIT(DBUF_S2),
 993			[PIPE_D] = BIT(DBUF_S2),
 994		},
 995	},
 996	{
 997		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
 998		.dbuf_mask = {
 999			[PIPE_B] = BIT(DBUF_S1),
1000			[PIPE_C] = BIT(DBUF_S2),
1001			[PIPE_D] = BIT(DBUF_S2),
1002		},
1003	},
1004	{
1005		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1006		.dbuf_mask = {
1007			[PIPE_A] = BIT(DBUF_S1),
1008			[PIPE_B] = BIT(DBUF_S1),
1009			[PIPE_C] = BIT(DBUF_S2),
1010			[PIPE_D] = BIT(DBUF_S2),
1011		},
1012	},
1013	{}
1014};
1015
1016static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1017	{
1018		.active_pipes = BIT(PIPE_A),
1019		.dbuf_mask = {
1020			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1021		},
1022	},
1023	{
1024		.active_pipes = BIT(PIPE_B),
1025		.dbuf_mask = {
1026			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027		},
1028	},
1029	{
1030		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1031		.dbuf_mask = {
1032			[PIPE_A] = BIT(DBUF_S1),
1033			[PIPE_B] = BIT(DBUF_S2),
1034		},
1035	},
1036	{
1037		.active_pipes = BIT(PIPE_C),
1038		.dbuf_mask = {
1039			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1040		},
1041	},
1042	{
1043		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1044		.dbuf_mask = {
1045			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1047		},
1048	},
1049	{
1050		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1051		.dbuf_mask = {
1052			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1053			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1054		},
1055	},
1056	{
1057		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1058		.dbuf_mask = {
1059			[PIPE_A] = BIT(DBUF_S1),
1060			[PIPE_B] = BIT(DBUF_S2),
1061			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1062		},
1063	},
1064	{
1065		.active_pipes = BIT(PIPE_D),
1066		.dbuf_mask = {
1067			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1068		},
1069	},
1070	{
1071		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1072		.dbuf_mask = {
1073			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1074			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1075		},
1076	},
1077	{
1078		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1079		.dbuf_mask = {
1080			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1081			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1082		},
1083	},
1084	{
1085		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1086		.dbuf_mask = {
1087			[PIPE_A] = BIT(DBUF_S1),
1088			[PIPE_B] = BIT(DBUF_S2),
1089			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1090		},
1091	},
1092	{
1093		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1094		.dbuf_mask = {
1095			[PIPE_C] = BIT(DBUF_S3),
1096			[PIPE_D] = BIT(DBUF_S4),
1097		},
1098	},
1099	{
1100		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1101		.dbuf_mask = {
1102			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1103			[PIPE_C] = BIT(DBUF_S3),
1104			[PIPE_D] = BIT(DBUF_S4),
1105		},
1106	},
1107	{
1108		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1109		.dbuf_mask = {
1110			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1111			[PIPE_C] = BIT(DBUF_S3),
1112			[PIPE_D] = BIT(DBUF_S4),
1113		},
1114	},
1115	{
1116		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1117		.dbuf_mask = {
1118			[PIPE_A] = BIT(DBUF_S1),
1119			[PIPE_B] = BIT(DBUF_S2),
1120			[PIPE_C] = BIT(DBUF_S3),
1121			[PIPE_D] = BIT(DBUF_S4),
1122		},
1123	},
1124	{}
1125};
1126
1127static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1128	/*
1129	 * Keep the join_mbus cases first so check_mbus_joined()
1130	 * will prefer them over the !join_mbus cases.
1131	 */
1132	{
1133		.active_pipes = BIT(PIPE_A),
1134		.dbuf_mask = {
1135			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1136		},
1137		.join_mbus = true,
1138	},
1139	{
1140		.active_pipes = BIT(PIPE_B),
1141		.dbuf_mask = {
1142			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1143		},
1144		.join_mbus = true,
1145	},
1146	{
1147		.active_pipes = BIT(PIPE_A),
1148		.dbuf_mask = {
1149			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150		},
1151		.join_mbus = false,
1152	},
1153	{
1154		.active_pipes = BIT(PIPE_B),
1155		.dbuf_mask = {
1156			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1157		},
1158		.join_mbus = false,
1159	},
1160	{
1161		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1162		.dbuf_mask = {
1163			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1164			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1165		},
1166	},
1167	{
1168		.active_pipes = BIT(PIPE_C),
1169		.dbuf_mask = {
1170			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1171		},
1172	},
1173	{
1174		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1175		.dbuf_mask = {
1176			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1177			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1178		},
1179	},
1180	{
1181		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1182		.dbuf_mask = {
1183			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1185		},
1186	},
1187	{
1188		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1189		.dbuf_mask = {
1190			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1191			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1192			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1193		},
1194	},
1195	{
1196		.active_pipes = BIT(PIPE_D),
1197		.dbuf_mask = {
1198			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1199		},
1200	},
1201	{
1202		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1203		.dbuf_mask = {
1204			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1205			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1206		},
1207	},
1208	{
1209		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1210		.dbuf_mask = {
1211			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1213		},
1214	},
1215	{
1216		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1217		.dbuf_mask = {
1218			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1219			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1220			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1221		},
1222	},
1223	{
1224		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1225		.dbuf_mask = {
1226			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1227			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1228		},
1229	},
1230	{
1231		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1232		.dbuf_mask = {
1233			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1234			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1235			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1236		},
1237	},
1238	{
1239		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1240		.dbuf_mask = {
1241			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1242			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1243			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244		},
1245	},
1246	{
1247		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1248		.dbuf_mask = {
1249			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1250			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1251			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1252			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253		},
1254	},
1255	{}
1256
1257};
1258
1259static bool check_mbus_joined(u8 active_pipes,
1260			      const struct dbuf_slice_conf_entry *dbuf_slices)
1261{
1262	int i;
1263
1264	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1265		if (dbuf_slices[i].active_pipes == active_pipes)
1266			return dbuf_slices[i].join_mbus;
1267	}
1268	return false;
1269}
1270
1271static bool adlp_check_mbus_joined(u8 active_pipes)
1272{
1273	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1274}
1275
1276static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1277			      const struct dbuf_slice_conf_entry *dbuf_slices)
1278{
1279	int i;
1280
1281	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1282		if (dbuf_slices[i].active_pipes == active_pipes &&
1283		    dbuf_slices[i].join_mbus == join_mbus)
1284			return dbuf_slices[i].dbuf_mask[pipe];
1285	}
1286	return 0;
1287}
1288
1289/*
1290 * This function finds an entry with same enabled pipe configuration and
1291 * returns correspondent DBuf slice mask as stated in BSpec for particular
1292 * platform.
1293 */
1294static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1295{
1296	/*
1297	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1298	 * required calculating "pipe ratio" in order to determine
1299	 * if one or two slices can be used for single pipe configurations
1300	 * as additional constraint to the existing table.
1301	 * However based on recent info, it should be not "pipe ratio"
1302	 * but rather ratio between pixel_rate and cdclk with additional
1303	 * constants, so for now we are using only table until this is
1304	 * clarified. Also this is the reason why crtc_state param is
1305	 * still here - we will need it once those additional constraints
1306	 * pop up.
1307	 */
1308	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1309				   icl_allowed_dbufs);
1310}
1311
1312static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1313{
1314	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1315				   tgl_allowed_dbufs);
1316}
1317
1318static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1319{
1320	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1321				   adlp_allowed_dbufs);
1322}
1323
1324static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1325{
1326	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1327				   dg2_allowed_dbufs);
1328}
1329
1330static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1331{
1332	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1333	enum pipe pipe = crtc->pipe;
1334
1335	if (IS_DG2(i915))
1336		return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1337	else if (DISPLAY_VER(i915) >= 13)
1338		return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1339	else if (DISPLAY_VER(i915) == 12)
1340		return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1341	else if (DISPLAY_VER(i915) == 11)
1342		return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1343	/*
1344	 * For anything else just return one slice yet.
1345	 * Should be extended for other platforms.
1346	 */
1347	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1348}
1349
1350static bool
1351use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1352		     struct intel_plane *plane)
1353{
1354	struct drm_i915_private *i915 = to_i915(plane->base.dev);
1355
1356	return DISPLAY_VER(i915) >= 13 &&
1357	       crtc_state->uapi.async_flip &&
1358	       plane->async_flip;
1359}
1360
1361static u64
1362skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1363{
1364	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1365	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1366	enum plane_id plane_id;
1367	u64 data_rate = 0;
1368
1369	for_each_plane_id_on_crtc(crtc, plane_id) {
1370		if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20)
1371			continue;
1372
1373		data_rate += crtc_state->rel_data_rate[plane_id];
1374
1375		if (DISPLAY_VER(i915) < 11)
1376			data_rate += crtc_state->rel_data_rate_y[plane_id];
1377	}
1378
1379	return data_rate;
1380}
1381
1382static const struct skl_wm_level *
1383skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1384		   enum plane_id plane_id,
1385		   int level)
1386{
1387	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1388
1389	if (level == 0 && pipe_wm->use_sagv_wm)
1390		return &wm->sagv.wm0;
1391
1392	return &wm->wm[level];
1393}
1394
1395static const struct skl_wm_level *
1396skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1397		   enum plane_id plane_id)
1398{
1399	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1400
1401	if (pipe_wm->use_sagv_wm)
1402		return &wm->sagv.trans_wm;
1403
1404	return &wm->trans_wm;
1405}
1406
1407/*
1408 * We only disable the watermarks for each plane if
1409 * they exceed the ddb allocation of said plane. This
1410 * is done so that we don't end up touching cursor
1411 * watermarks needlessly when some other plane reduces
1412 * our max possible watermark level.
1413 *
1414 * Bspec has this to say about the PLANE_WM enable bit:
1415 * "All the watermarks at this level for all enabled
1416 *  planes must be enabled before the level will be used."
1417 * So this is actually safe to do.
1418 */
1419static void
1420skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1421{
1422	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1423		memset(wm, 0, sizeof(*wm));
1424}
1425
1426static void
1427skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1428			const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1429{
1430	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1431	    uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1432		memset(wm, 0, sizeof(*wm));
1433		memset(uv_wm, 0, sizeof(*uv_wm));
1434	}
1435}
1436
1437static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1438				const struct skl_plane_wm *wm)
1439{
1440	/*
1441	 * Wa_1408961008:icl, ehl
1442	 * Wa_14012656716:tgl, adl
1443	 * Wa_14017887344:icl
1444	 * Wa_14017868169:adl, tgl
1445	 * Due to some power saving optimizations, different subsystems
1446	 * like PSR, might still use even disabled wm level registers,
1447	 * for "reference", so lets keep at least the values sane.
1448	 * Considering amount of WA requiring us to do similar things, was
1449	 * decided to simply do it for all of the platforms, as those wm
1450	 * levels are disabled, this isn't going to do harm anyway.
1451	 */
1452	return level > 0 && !wm->wm[level].enable;
1453}
1454
1455struct skl_plane_ddb_iter {
1456	u64 data_rate;
1457	u16 start, size;
1458};
1459
1460static void
1461skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1462		       struct skl_ddb_entry *ddb,
1463		       const struct skl_wm_level *wm,
1464		       u64 data_rate)
1465{
1466	u16 size, extra = 0;
1467
1468	if (data_rate) {
1469		extra = min_t(u16, iter->size,
1470			      DIV64_U64_ROUND_UP(iter->size * data_rate,
1471						 iter->data_rate));
1472		iter->size -= extra;
1473		iter->data_rate -= data_rate;
1474	}
1475
1476	/*
1477	 * Keep ddb entry of all disabled planes explicitly zeroed
1478	 * to avoid skl_ddb_add_affected_planes() adding them to
1479	 * the state when other planes change their allocations.
1480	 */
1481	size = wm->min_ddb_alloc + extra;
1482	if (size)
1483		iter->start = skl_ddb_entry_init(ddb, iter->start,
1484						 iter->start + size);
1485}
1486
1487static int
1488skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1489			    struct intel_crtc *crtc)
1490{
1491	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1492	struct intel_crtc_state *crtc_state =
1493		intel_atomic_get_new_crtc_state(state, crtc);
1494	const struct intel_dbuf_state *dbuf_state =
1495		intel_atomic_get_new_dbuf_state(state);
1496	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1497	int num_active = hweight8(dbuf_state->active_pipes);
1498	struct skl_plane_ddb_iter iter;
1499	enum plane_id plane_id;
1500	u16 cursor_size;
1501	u32 blocks;
1502	int level;
1503
1504	/* Clear the partitioning for disabled planes. */
1505	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1506	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1507
1508	if (!crtc_state->hw.active)
1509		return 0;
1510
1511	iter.start = alloc->start;
1512	iter.size = skl_ddb_entry_size(alloc);
1513	if (iter.size == 0)
1514		return 0;
1515
1516	/* Allocate fixed number of blocks for cursor. */
1517	if (DISPLAY_VER(i915) < 20) {
1518		cursor_size = skl_cursor_allocation(crtc_state, num_active);
1519		iter.size -= cursor_size;
1520		skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1521				   alloc->end - cursor_size, alloc->end);
1522	}
1523
1524	iter.data_rate = skl_total_relative_data_rate(crtc_state);
1525
1526	/*
1527	 * Find the highest watermark level for which we can satisfy the block
1528	 * requirement of active planes.
1529	 */
1530	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1531		blocks = 0;
1532		for_each_plane_id_on_crtc(crtc, plane_id) {
1533			const struct skl_plane_wm *wm =
1534				&crtc_state->wm.skl.optimal.planes[plane_id];
1535
1536			if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20) {
1537				const struct skl_ddb_entry *ddb =
1538					&crtc_state->wm.skl.plane_ddb[plane_id];
1539
1540				if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1541					drm_WARN_ON(&i915->drm,
1542						    wm->wm[level].min_ddb_alloc != U16_MAX);
1543					blocks = U32_MAX;
1544					break;
1545				}
1546				continue;
1547			}
1548
1549			blocks += wm->wm[level].min_ddb_alloc;
1550			blocks += wm->uv_wm[level].min_ddb_alloc;
1551		}
1552
1553		if (blocks <= iter.size) {
1554			iter.size -= blocks;
1555			break;
1556		}
1557	}
1558
1559	if (level < 0) {
1560		drm_dbg_kms(&i915->drm,
1561			    "Requested display configuration exceeds system DDB limitations");
1562		drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1563			    blocks, iter.size);
1564		return -EINVAL;
1565	}
1566
1567	/* avoid the WARN later when we don't allocate any extra DDB */
1568	if (iter.data_rate == 0)
1569		iter.size = 0;
1570
1571	/*
1572	 * Grant each plane the blocks it requires at the highest achievable
1573	 * watermark level, plus an extra share of the leftover blocks
1574	 * proportional to its relative data rate.
1575	 */
1576	for_each_plane_id_on_crtc(crtc, plane_id) {
1577		struct skl_ddb_entry *ddb =
1578			&crtc_state->wm.skl.plane_ddb[plane_id];
1579		struct skl_ddb_entry *ddb_y =
1580			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1581		const struct skl_plane_wm *wm =
1582			&crtc_state->wm.skl.optimal.planes[plane_id];
1583
1584		if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20)
1585			continue;
1586
1587		if (DISPLAY_VER(i915) < 11 &&
1588		    crtc_state->nv12_planes & BIT(plane_id)) {
1589			skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1590					       crtc_state->rel_data_rate_y[plane_id]);
1591			skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1592					       crtc_state->rel_data_rate[plane_id]);
1593		} else {
1594			skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1595					       crtc_state->rel_data_rate[plane_id]);
1596		}
1597	}
1598	drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1599
1600	/*
1601	 * When we calculated watermark values we didn't know how high
1602	 * of a level we'd actually be able to hit, so we just marked
1603	 * all levels as "enabled."  Go back now and disable the ones
1604	 * that aren't actually possible.
1605	 */
1606	for (level++; level < i915->display.wm.num_levels; level++) {
1607		for_each_plane_id_on_crtc(crtc, plane_id) {
1608			const struct skl_ddb_entry *ddb =
1609				&crtc_state->wm.skl.plane_ddb[plane_id];
1610			const struct skl_ddb_entry *ddb_y =
1611				&crtc_state->wm.skl.plane_ddb_y[plane_id];
1612			struct skl_plane_wm *wm =
1613				&crtc_state->wm.skl.optimal.planes[plane_id];
1614
1615			if (DISPLAY_VER(i915) < 11 &&
1616			    crtc_state->nv12_planes & BIT(plane_id))
1617				skl_check_nv12_wm_level(&wm->wm[level],
1618							&wm->uv_wm[level],
1619							ddb_y, ddb);
1620			else
1621				skl_check_wm_level(&wm->wm[level], ddb);
1622
1623			if (skl_need_wm_copy_wa(i915, level, wm)) {
1624				wm->wm[level].blocks = wm->wm[level - 1].blocks;
1625				wm->wm[level].lines = wm->wm[level - 1].lines;
1626				wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1627			}
1628		}
1629	}
1630
1631	/*
1632	 * Go back and disable the transition and SAGV watermarks
1633	 * if it turns out we don't have enough DDB blocks for them.
1634	 */
1635	for_each_plane_id_on_crtc(crtc, plane_id) {
1636		const struct skl_ddb_entry *ddb =
1637			&crtc_state->wm.skl.plane_ddb[plane_id];
1638		const struct skl_ddb_entry *ddb_y =
1639			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1640		struct skl_plane_wm *wm =
1641			&crtc_state->wm.skl.optimal.planes[plane_id];
1642
1643		if (DISPLAY_VER(i915) < 11 &&
1644		    crtc_state->nv12_planes & BIT(plane_id)) {
1645			skl_check_wm_level(&wm->trans_wm, ddb_y);
1646		} else {
1647			WARN_ON(skl_ddb_entry_size(ddb_y));
1648
1649			skl_check_wm_level(&wm->trans_wm, ddb);
1650		}
1651
1652		skl_check_wm_level(&wm->sagv.wm0, ddb);
1653		skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1654	}
1655
1656	return 0;
1657}
1658
1659/*
1660 * The max latency should be 257 (max the punit can code is 255 and we add 2us
1661 * for the read latency) and cpp should always be <= 8, so that
1662 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1663 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1664 */
1665static uint_fixed_16_16_t
1666skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1667	       u8 cpp, u32 latency, u32 dbuf_block_size)
1668{
1669	u32 wm_intermediate_val;
1670	uint_fixed_16_16_t ret;
1671
1672	if (latency == 0)
1673		return FP_16_16_MAX;
1674
1675	wm_intermediate_val = latency * pixel_rate * cpp;
1676	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1677
1678	if (DISPLAY_VER(i915) >= 10)
1679		ret = add_fixed16_u32(ret, 1);
1680
1681	return ret;
1682}
1683
1684static uint_fixed_16_16_t
1685skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1686	       uint_fixed_16_16_t plane_blocks_per_line)
1687{
1688	u32 wm_intermediate_val;
1689	uint_fixed_16_16_t ret;
1690
1691	if (latency == 0)
1692		return FP_16_16_MAX;
1693
1694	wm_intermediate_val = latency * pixel_rate;
1695	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1696					   pipe_htotal * 1000);
1697	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1698	return ret;
1699}
1700
1701static uint_fixed_16_16_t
1702intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1703{
1704	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1705	u32 pixel_rate;
1706	u32 crtc_htotal;
1707	uint_fixed_16_16_t linetime_us;
1708
1709	if (!crtc_state->hw.active)
1710		return u32_to_fixed16(0);
1711
1712	pixel_rate = crtc_state->pixel_rate;
1713
1714	if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1715		return u32_to_fixed16(0);
1716
1717	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1718	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1719
1720	return linetime_us;
1721}
1722
1723static int
1724skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1725		      int width, const struct drm_format_info *format,
1726		      u64 modifier, unsigned int rotation,
1727		      u32 plane_pixel_rate, struct skl_wm_params *wp,
1728		      int color_plane)
1729{
1730	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1731	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1732	u32 interm_pbpl;
1733
1734	/* only planar format has two planes */
1735	if (color_plane == 1 &&
1736	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1737		drm_dbg_kms(&i915->drm,
1738			    "Non planar format have single plane\n");
1739		return -EINVAL;
1740	}
1741
1742	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1743	wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1744		intel_fb_is_tiled_modifier(modifier);
1745	wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1746	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1747
1748	wp->width = width;
1749	if (color_plane == 1 && wp->is_planar)
1750		wp->width /= 2;
1751
1752	wp->cpp = format->cpp[color_plane];
1753	wp->plane_pixel_rate = plane_pixel_rate;
1754
1755	if (DISPLAY_VER(i915) >= 11 &&
1756	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1757		wp->dbuf_block_size = 256;
1758	else
1759		wp->dbuf_block_size = 512;
1760
1761	if (drm_rotation_90_or_270(rotation)) {
1762		switch (wp->cpp) {
1763		case 1:
1764			wp->y_min_scanlines = 16;
1765			break;
1766		case 2:
1767			wp->y_min_scanlines = 8;
1768			break;
1769		case 4:
1770			wp->y_min_scanlines = 4;
1771			break;
1772		default:
1773			MISSING_CASE(wp->cpp);
1774			return -EINVAL;
1775		}
1776	} else {
1777		wp->y_min_scanlines = 4;
1778	}
1779
1780	if (skl_needs_memory_bw_wa(i915))
1781		wp->y_min_scanlines *= 2;
1782
1783	wp->plane_bytes_per_line = wp->width * wp->cpp;
1784	if (wp->y_tiled) {
1785		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1786					   wp->y_min_scanlines,
1787					   wp->dbuf_block_size);
1788
1789		if (DISPLAY_VER(i915) >= 10)
1790			interm_pbpl++;
1791
1792		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1793							wp->y_min_scanlines);
1794	} else {
1795		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1796					   wp->dbuf_block_size);
1797
1798		if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1799			interm_pbpl++;
1800
1801		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1802	}
1803
1804	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1805					     wp->plane_blocks_per_line);
1806
1807	wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1808
1809	return 0;
1810}
1811
1812static int
1813skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1814			    const struct intel_plane_state *plane_state,
1815			    struct skl_wm_params *wp, int color_plane)
1816{
1817	const struct drm_framebuffer *fb = plane_state->hw.fb;
1818	int width;
1819
1820	/*
1821	 * Src coordinates are already rotated by 270 degrees for
1822	 * the 90/270 degree plane rotation cases (to match the
1823	 * GTT mapping), hence no need to account for rotation here.
1824	 */
1825	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1826
1827	return skl_compute_wm_params(crtc_state, width,
1828				     fb->format, fb->modifier,
1829				     plane_state->hw.rotation,
1830				     intel_plane_pixel_rate(crtc_state, plane_state),
1831				     wp, color_plane);
1832}
1833
1834static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1835{
1836	if (DISPLAY_VER(i915) >= 10)
1837		return true;
1838
1839	/* The number of lines are ignored for the level 0 watermark. */
1840	return level > 0;
1841}
1842
1843static int skl_wm_max_lines(struct drm_i915_private *i915)
1844{
1845	if (DISPLAY_VER(i915) >= 13)
1846		return 255;
1847	else
1848		return 31;
1849}
1850
1851static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1852				 struct intel_plane *plane,
1853				 int level,
1854				 unsigned int latency,
1855				 const struct skl_wm_params *wp,
1856				 const struct skl_wm_level *result_prev,
1857				 struct skl_wm_level *result /* out */)
1858{
1859	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1860	uint_fixed_16_16_t method1, method2;
1861	uint_fixed_16_16_t selected_result;
1862	u32 blocks, lines, min_ddb_alloc = 0;
1863
1864	if (latency == 0 ||
1865	    (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1866		/* reject it */
1867		result->min_ddb_alloc = U16_MAX;
1868		return;
1869	}
1870
1871	method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1872				 wp->cpp, latency, wp->dbuf_block_size);
1873	method2 = skl_wm_method2(wp->plane_pixel_rate,
1874				 crtc_state->hw.pipe_mode.crtc_htotal,
1875				 latency,
1876				 wp->plane_blocks_per_line);
1877
1878	if (wp->y_tiled) {
1879		selected_result = max_fixed16(method2, wp->y_tile_minimum);
1880	} else {
1881		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1882		     wp->dbuf_block_size < 1) &&
1883		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1884			selected_result = method2;
1885		} else if (latency >= wp->linetime_us) {
1886			if (DISPLAY_VER(i915) == 9)
1887				selected_result = min_fixed16(method1, method2);
1888			else
1889				selected_result = method2;
1890		} else {
1891			selected_result = method1;
1892		}
1893	}
1894
1895	blocks = fixed16_to_u32_round_up(selected_result) + 1;
1896	/*
1897	 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1898	 * as there will be at minimum one line for lines configuration. This
1899	 * is a work around for FIFO underruns observed with resolutions like
1900	 * 4k 60 Hz in single channel DRAM configurations.
1901	 *
1902	 * As per the Bspec 49325, if the ddb allocation can hold at least
1903	 * one plane_blocks_per_line, we should have selected method2 in
1904	 * the above logic. Assuming that modern versions have enough dbuf
1905	 * and method2 guarantees blocks equivalent to at least 1 line,
1906	 * select the blocks as plane_blocks_per_line.
1907	 *
1908	 * TODO: Revisit the logic when we have better understanding on DRAM
1909	 * channels' impact on the level 0 memory latency and the relevant
1910	 * wm calculations.
1911	 */
1912	if (skl_wm_has_lines(i915, level))
1913		blocks = max(blocks,
1914			     fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1915	lines = div_round_up_fixed16(selected_result,
1916				     wp->plane_blocks_per_line);
1917
1918	if (DISPLAY_VER(i915) == 9) {
1919		/* Display WA #1125: skl,bxt,kbl */
1920		if (level == 0 && wp->rc_surface)
1921			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1922
1923		/* Display WA #1126: skl,bxt,kbl */
1924		if (level >= 1 && level <= 7) {
1925			if (wp->y_tiled) {
1926				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1927				lines += wp->y_min_scanlines;
1928			} else {
1929				blocks++;
1930			}
1931
1932			/*
1933			 * Make sure result blocks for higher latency levels are
1934			 * at least as high as level below the current level.
1935			 * Assumption in DDB algorithm optimization for special
1936			 * cases. Also covers Display WA #1125 for RC.
1937			 */
1938			if (result_prev->blocks > blocks)
1939				blocks = result_prev->blocks;
1940		}
1941	}
1942
1943	if (DISPLAY_VER(i915) >= 11) {
1944		if (wp->y_tiled) {
1945			int extra_lines;
1946
1947			if (lines % wp->y_min_scanlines == 0)
1948				extra_lines = wp->y_min_scanlines;
1949			else
1950				extra_lines = wp->y_min_scanlines * 2 -
1951					lines % wp->y_min_scanlines;
1952
1953			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1954								 wp->plane_blocks_per_line);
1955		} else {
1956			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1957		}
1958	}
1959
1960	if (!skl_wm_has_lines(i915, level))
1961		lines = 0;
1962
1963	if (lines > skl_wm_max_lines(i915)) {
1964		/* reject it */
1965		result->min_ddb_alloc = U16_MAX;
1966		return;
1967	}
1968
1969	/*
1970	 * If lines is valid, assume we can use this watermark level
1971	 * for now.  We'll come back and disable it after we calculate the
1972	 * DDB allocation if it turns out we don't actually have enough
1973	 * blocks to satisfy it.
1974	 */
1975	result->blocks = blocks;
1976	result->lines = lines;
1977	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1978	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1979	result->enable = true;
1980
1981	if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1982		result->can_sagv = latency >= i915->display.sagv.block_time_us;
1983}
1984
1985static void
1986skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1987		      struct intel_plane *plane,
1988		      const struct skl_wm_params *wm_params,
1989		      struct skl_wm_level *levels)
1990{
1991	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1992	struct skl_wm_level *result_prev = &levels[0];
1993	int level;
1994
1995	for (level = 0; level < i915->display.wm.num_levels; level++) {
1996		struct skl_wm_level *result = &levels[level];
1997		unsigned int latency = skl_wm_latency(i915, level, wm_params);
1998
1999		skl_compute_plane_wm(crtc_state, plane, level, latency,
2000				     wm_params, result_prev, result);
2001
2002		result_prev = result;
2003	}
2004}
2005
2006static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2007				struct intel_plane *plane,
2008				const struct skl_wm_params *wm_params,
2009				struct skl_plane_wm *plane_wm)
2010{
2011	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2012	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2013	struct skl_wm_level *levels = plane_wm->wm;
2014	unsigned int latency = 0;
2015
2016	if (i915->display.sagv.block_time_us)
2017		latency = i915->display.sagv.block_time_us +
2018			skl_wm_latency(i915, 0, wm_params);
2019
2020	skl_compute_plane_wm(crtc_state, plane, 0, latency,
2021			     wm_params, &levels[0],
2022			     sagv_wm);
2023}
2024
2025static void skl_compute_transition_wm(struct drm_i915_private *i915,
2026				      struct skl_wm_level *trans_wm,
2027				      const struct skl_wm_level *wm0,
2028				      const struct skl_wm_params *wp)
2029{
2030	u16 trans_min, trans_amount, trans_y_tile_min;
2031	u16 wm0_blocks, trans_offset, blocks;
2032
2033	/* Transition WM don't make any sense if ipc is disabled */
2034	if (!skl_watermark_ipc_enabled(i915))
2035		return;
2036
2037	/*
2038	 * WaDisableTWM:skl,kbl,cfl,bxt
2039	 * Transition WM are not recommended by HW team for GEN9
2040	 */
2041	if (DISPLAY_VER(i915) == 9)
2042		return;
2043
2044	if (DISPLAY_VER(i915) >= 11)
2045		trans_min = 4;
2046	else
2047		trans_min = 14;
2048
2049	/* Display WA #1140: glk,cnl */
2050	if (DISPLAY_VER(i915) == 10)
2051		trans_amount = 0;
2052	else
2053		trans_amount = 10; /* This is configurable amount */
2054
2055	trans_offset = trans_min + trans_amount;
2056
2057	/*
2058	 * The spec asks for Selected Result Blocks for wm0 (the real value),
2059	 * not Result Blocks (the integer value). Pay attention to the capital
2060	 * letters. The value wm_l0->blocks is actually Result Blocks, but
2061	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2062	 * and since we later will have to get the ceiling of the sum in the
2063	 * transition watermarks calculation, we can just pretend Selected
2064	 * Result Blocks is Result Blocks minus 1 and it should work for the
2065	 * current platforms.
2066	 */
2067	wm0_blocks = wm0->blocks - 1;
2068
2069	if (wp->y_tiled) {
2070		trans_y_tile_min =
2071			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2072		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2073	} else {
2074		blocks = wm0_blocks + trans_offset;
2075	}
2076	blocks++;
2077
2078	/*
2079	 * Just assume we can enable the transition watermark.  After
2080	 * computing the DDB we'll come back and disable it if that
2081	 * assumption turns out to be false.
2082	 */
2083	trans_wm->blocks = blocks;
2084	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2085	trans_wm->enable = true;
2086}
2087
2088static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2089				     const struct intel_plane_state *plane_state,
2090				     struct intel_plane *plane, int color_plane)
2091{
2092	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2093	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2094	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2095	struct skl_wm_params wm_params;
2096	int ret;
2097
2098	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2099					  &wm_params, color_plane);
2100	if (ret)
2101		return ret;
2102
2103	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2104
2105	skl_compute_transition_wm(i915, &wm->trans_wm,
2106				  &wm->wm[0], &wm_params);
2107
2108	if (DISPLAY_VER(i915) >= 12) {
2109		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2110
2111		skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2112					  &wm->sagv.wm0, &wm_params);
2113	}
2114
2115	return 0;
2116}
2117
2118static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2119				 const struct intel_plane_state *plane_state,
2120				 struct intel_plane *plane)
2121{
2122	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2123	struct skl_wm_params wm_params;
2124	int ret;
2125
2126	wm->is_planar = true;
2127
2128	/* uv plane watermarks must also be validated for NV12/Planar */
2129	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2130					  &wm_params, 1);
2131	if (ret)
2132		return ret;
2133
2134	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2135
2136	return 0;
2137}
2138
2139static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2140			      const struct intel_plane_state *plane_state)
2141{
2142	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2143	enum plane_id plane_id = plane->id;
2144	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2145	const struct drm_framebuffer *fb = plane_state->hw.fb;
2146	int ret;
2147
2148	memset(wm, 0, sizeof(*wm));
2149
2150	if (!intel_wm_plane_visible(crtc_state, plane_state))
2151		return 0;
2152
2153	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2154					plane, 0);
2155	if (ret)
2156		return ret;
2157
2158	if (fb->format->is_yuv && fb->format->num_planes > 1) {
2159		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2160					    plane);
2161		if (ret)
2162			return ret;
2163	}
2164
2165	return 0;
2166}
2167
2168static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2169			      const struct intel_plane_state *plane_state)
2170{
2171	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2172	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2173	enum plane_id plane_id = plane->id;
2174	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2175	int ret;
2176
2177	/* Watermarks calculated in master */
2178	if (plane_state->planar_slave)
2179		return 0;
2180
2181	memset(wm, 0, sizeof(*wm));
2182
2183	if (plane_state->planar_linked_plane) {
2184		const struct drm_framebuffer *fb = plane_state->hw.fb;
2185
2186		drm_WARN_ON(&i915->drm,
2187			    !intel_wm_plane_visible(crtc_state, plane_state));
2188		drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2189			    fb->format->num_planes == 1);
2190
2191		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2192						plane_state->planar_linked_plane, 0);
2193		if (ret)
2194			return ret;
2195
2196		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2197						plane, 1);
2198		if (ret)
2199			return ret;
2200	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2201		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2202						plane, 0);
2203		if (ret)
2204			return ret;
2205	}
2206
2207	return 0;
2208}
2209
2210static bool
2211skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2212			int wm0_lines, int latency)
2213{
2214	const struct drm_display_mode *adjusted_mode =
2215		&crtc_state->hw.adjusted_mode;
2216
2217	/* FIXME missing scaler and DSC pre-fill time */
2218	return crtc_state->framestart_delay +
2219		intel_usecs_to_scanlines(adjusted_mode, latency) +
2220		wm0_lines >
2221		adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2222}
2223
2224static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2225{
2226	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2227	enum plane_id plane_id;
2228	int wm0_lines = 0;
2229
2230	for_each_plane_id_on_crtc(crtc, plane_id) {
2231		const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2232
2233		/* FIXME what about !skl_wm_has_lines() platforms? */
2234		wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2235	}
2236
2237	return wm0_lines;
2238}
2239
2240static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2241				       int wm0_lines)
2242{
2243	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2244	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2245	int level;
2246
2247	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2248		int latency;
2249
2250		/* FIXME should we care about the latency w/a's? */
2251		latency = skl_wm_latency(i915, level, NULL);
2252		if (latency == 0)
2253			continue;
2254
2255		/* FIXME is it correct to use 0 latency for wm0 here? */
2256		if (level == 0)
2257			latency = 0;
2258
2259		if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2260			return level;
2261	}
2262
2263	return -EINVAL;
2264}
2265
2266static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2267{
2268	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2269	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2270	int wm0_lines, level;
2271
2272	if (!crtc_state->hw.active)
2273		return 0;
2274
2275	wm0_lines = skl_max_wm0_lines(crtc_state);
2276
2277	level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2278	if (level < 0)
2279		return level;
2280
2281	/*
2282	 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2283	 * based on whether we're limited by the vblank duration.
2284	 */
2285	crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2286
2287	for (level++; level < i915->display.wm.num_levels; level++) {
2288		enum plane_id plane_id;
2289
2290		for_each_plane_id_on_crtc(crtc, plane_id) {
2291			struct skl_plane_wm *wm =
2292				&crtc_state->wm.skl.optimal.planes[plane_id];
2293
2294			/*
2295			 * FIXME just clear enable or flag the entire
2296			 * thing as bad via min_ddb_alloc=U16_MAX?
2297			 */
2298			wm->wm[level].enable = false;
2299			wm->uv_wm[level].enable = false;
2300		}
2301	}
2302
2303	if (DISPLAY_VER(i915) >= 12 &&
2304	    i915->display.sagv.block_time_us &&
2305	    skl_is_vblank_too_short(crtc_state, wm0_lines,
2306				    i915->display.sagv.block_time_us)) {
2307		enum plane_id plane_id;
2308
2309		for_each_plane_id_on_crtc(crtc, plane_id) {
2310			struct skl_plane_wm *wm =
2311				&crtc_state->wm.skl.optimal.planes[plane_id];
2312
2313			wm->sagv.wm0.enable = false;
2314			wm->sagv.trans_wm.enable = false;
2315		}
2316	}
2317
2318	return 0;
2319}
2320
2321static int skl_build_pipe_wm(struct intel_atomic_state *state,
2322			     struct intel_crtc *crtc)
2323{
2324	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2325	struct intel_crtc_state *crtc_state =
2326		intel_atomic_get_new_crtc_state(state, crtc);
2327	const struct intel_plane_state *plane_state;
2328	struct intel_plane *plane;
2329	int ret, i;
2330
2331	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2332		/*
2333		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2334		 * instead but we don't populate that correctly for NV12 Y
2335		 * planes so for now hack this.
2336		 */
2337		if (plane->pipe != crtc->pipe)
2338			continue;
2339
2340		if (DISPLAY_VER(i915) >= 11)
2341			ret = icl_build_plane_wm(crtc_state, plane_state);
2342		else
2343			ret = skl_build_plane_wm(crtc_state, plane_state);
2344		if (ret)
2345			return ret;
2346	}
2347
2348	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2349
2350	return skl_wm_check_vblank(crtc_state);
2351}
2352
2353static void skl_ddb_entry_write(struct drm_i915_private *i915,
2354				i915_reg_t reg,
2355				const struct skl_ddb_entry *entry)
2356{
2357	if (entry->end)
2358		intel_de_write_fw(i915, reg,
2359				  PLANE_BUF_END(entry->end - 1) |
2360				  PLANE_BUF_START(entry->start));
2361	else
2362		intel_de_write_fw(i915, reg, 0);
2363}
2364
2365static void skl_write_wm_level(struct drm_i915_private *i915,
2366			       i915_reg_t reg,
2367			       const struct skl_wm_level *level)
2368{
2369	u32 val = 0;
2370
2371	if (level->enable)
2372		val |= PLANE_WM_EN;
2373	if (level->ignore_lines)
2374		val |= PLANE_WM_IGNORE_LINES;
2375	val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2376	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2377
2378	intel_de_write_fw(i915, reg, val);
2379}
2380
2381void skl_write_plane_wm(struct intel_plane *plane,
2382			const struct intel_crtc_state *crtc_state)
2383{
2384	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2385	enum plane_id plane_id = plane->id;
2386	enum pipe pipe = plane->pipe;
2387	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2388	const struct skl_ddb_entry *ddb =
2389		&crtc_state->wm.skl.plane_ddb[plane_id];
2390	const struct skl_ddb_entry *ddb_y =
2391		&crtc_state->wm.skl.plane_ddb_y[plane_id];
2392	int level;
2393
2394	for (level = 0; level < i915->display.wm.num_levels; level++)
2395		skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2396				   skl_plane_wm_level(pipe_wm, plane_id, level));
2397
2398	skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2399			   skl_plane_trans_wm(pipe_wm, plane_id));
2400
2401	if (HAS_HW_SAGV_WM(i915)) {
2402		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2403
2404		skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2405				   &wm->sagv.wm0);
2406		skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2407				   &wm->sagv.trans_wm);
2408	}
2409
2410	skl_ddb_entry_write(i915,
2411			    PLANE_BUF_CFG(pipe, plane_id), ddb);
2412
2413	if (DISPLAY_VER(i915) < 11)
2414		skl_ddb_entry_write(i915,
2415				    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
2416}
2417
2418void skl_write_cursor_wm(struct intel_plane *plane,
2419			 const struct intel_crtc_state *crtc_state)
2420{
2421	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2422	enum plane_id plane_id = plane->id;
2423	enum pipe pipe = plane->pipe;
2424	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2425	const struct skl_ddb_entry *ddb =
2426		&crtc_state->wm.skl.plane_ddb[plane_id];
2427	int level;
2428
2429	for (level = 0; level < i915->display.wm.num_levels; level++)
2430		skl_write_wm_level(i915, CUR_WM(pipe, level),
2431				   skl_plane_wm_level(pipe_wm, plane_id, level));
2432
2433	skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2434			   skl_plane_trans_wm(pipe_wm, plane_id));
2435
2436	if (HAS_HW_SAGV_WM(i915)) {
2437		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2438
2439		skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2440				   &wm->sagv.wm0);
2441		skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2442				   &wm->sagv.trans_wm);
2443	}
2444
2445	skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
2446}
2447
2448static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2449				const struct skl_wm_level *l2)
2450{
2451	return l1->enable == l2->enable &&
2452		l1->ignore_lines == l2->ignore_lines &&
2453		l1->lines == l2->lines &&
2454		l1->blocks == l2->blocks;
2455}
2456
2457static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2458				const struct skl_plane_wm *wm1,
2459				const struct skl_plane_wm *wm2)
2460{
2461	int level;
2462
2463	for (level = 0; level < i915->display.wm.num_levels; level++) {
2464		/*
2465		 * We don't check uv_wm as the hardware doesn't actually
2466		 * use it. It only gets used for calculating the required
2467		 * ddb allocation.
2468		 */
2469		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2470			return false;
2471	}
2472
2473	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2474		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2475		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2476}
2477
2478static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2479				    const struct skl_ddb_entry *b)
2480{
2481	return a->start < b->end && b->start < a->end;
2482}
2483
2484static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2485				const struct skl_ddb_entry *b)
2486{
2487	if (a->end && b->end) {
2488		a->start = min(a->start, b->start);
2489		a->end = max(a->end, b->end);
2490	} else if (b->end) {
2491		a->start = b->start;
2492		a->end = b->end;
2493	}
2494}
2495
2496bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2497				 const struct skl_ddb_entry *entries,
2498				 int num_entries, int ignore_idx)
2499{
2500	int i;
2501
2502	for (i = 0; i < num_entries; i++) {
2503		if (i != ignore_idx &&
2504		    skl_ddb_entries_overlap(ddb, &entries[i]))
2505			return true;
2506	}
2507
2508	return false;
2509}
2510
2511static int
2512skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2513			    struct intel_crtc_state *new_crtc_state)
2514{
2515	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2516	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2517	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2518	struct intel_plane *plane;
2519
2520	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2521		struct intel_plane_state *plane_state;
2522		enum plane_id plane_id = plane->id;
2523
2524		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2525					&new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2526		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2527					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2528			continue;
2529
2530		plane_state = intel_atomic_get_plane_state(state, plane);
2531		if (IS_ERR(plane_state))
2532			return PTR_ERR(plane_state);
2533
2534		new_crtc_state->update_planes |= BIT(plane_id);
2535		new_crtc_state->async_flip_planes = 0;
2536		new_crtc_state->do_async_flip = false;
2537	}
2538
2539	return 0;
2540}
2541
2542static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2543{
2544	struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2545	u8 enabled_slices;
2546	enum pipe pipe;
2547
2548	/*
2549	 * FIXME: For now we always enable slice S1 as per
2550	 * the Bspec display initialization sequence.
2551	 */
2552	enabled_slices = BIT(DBUF_S1);
2553
2554	for_each_pipe(i915, pipe)
2555		enabled_slices |= dbuf_state->slices[pipe];
2556
2557	return enabled_slices;
2558}
2559
2560static int
2561skl_compute_ddb(struct intel_atomic_state *state)
2562{
2563	struct drm_i915_private *i915 = to_i915(state->base.dev);
2564	const struct intel_dbuf_state *old_dbuf_state;
2565	struct intel_dbuf_state *new_dbuf_state = NULL;
2566	const struct intel_crtc_state *old_crtc_state;
2567	struct intel_crtc_state *new_crtc_state;
2568	struct intel_crtc *crtc;
2569	int ret, i;
2570
2571	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2572		new_dbuf_state = intel_atomic_get_dbuf_state(state);
2573		if (IS_ERR(new_dbuf_state))
2574			return PTR_ERR(new_dbuf_state);
2575
2576		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2577		break;
2578	}
2579
2580	if (!new_dbuf_state)
2581		return 0;
2582
2583	new_dbuf_state->active_pipes =
2584		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2585
2586	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2587		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2588		if (ret)
2589			return ret;
2590	}
2591
2592	if (HAS_MBUS_JOINING(i915))
2593		new_dbuf_state->joined_mbus =
2594			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2595
2596	for_each_intel_crtc(&i915->drm, crtc) {
2597		enum pipe pipe = crtc->pipe;
2598
2599		new_dbuf_state->slices[pipe] =
2600			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2601						new_dbuf_state->joined_mbus);
2602
2603		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2604			continue;
2605
2606		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2607		if (ret)
2608			return ret;
2609	}
2610
2611	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2612
2613	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2614	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2615		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2616		if (ret)
2617			return ret;
2618
2619		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2620			/* TODO: Implement vblank synchronized MBUS joining changes */
2621			ret = intel_modeset_all_pipes_late(state, "MBUS joining change");
2622			if (ret)
2623				return ret;
2624		}
2625
2626		drm_dbg_kms(&i915->drm,
2627			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2628			    old_dbuf_state->enabled_slices,
2629			    new_dbuf_state->enabled_slices,
2630			    DISPLAY_INFO(i915)->dbuf.slice_mask,
2631			    str_yes_no(old_dbuf_state->joined_mbus),
2632			    str_yes_no(new_dbuf_state->joined_mbus));
2633	}
2634
2635	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2636		enum pipe pipe = crtc->pipe;
2637
2638		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2639
2640		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2641			continue;
2642
2643		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2644		if (ret)
2645			return ret;
2646	}
2647
2648	for_each_intel_crtc(&i915->drm, crtc) {
2649		ret = skl_crtc_allocate_ddb(state, crtc);
2650		if (ret)
2651			return ret;
2652	}
2653
2654	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2655					    new_crtc_state, i) {
2656		ret = skl_crtc_allocate_plane_ddb(state, crtc);
2657		if (ret)
2658			return ret;
2659
2660		ret = skl_ddb_add_affected_planes(old_crtc_state,
2661						  new_crtc_state);
2662		if (ret)
2663			return ret;
2664	}
2665
2666	return 0;
2667}
2668
2669static char enast(bool enable)
2670{
2671	return enable ? '*' : ' ';
2672}
2673
2674static void
2675skl_print_wm_changes(struct intel_atomic_state *state)
2676{
2677	struct drm_i915_private *i915 = to_i915(state->base.dev);
2678	const struct intel_crtc_state *old_crtc_state;
2679	const struct intel_crtc_state *new_crtc_state;
2680	struct intel_plane *plane;
2681	struct intel_crtc *crtc;
2682	int i;
2683
2684	if (!drm_debug_enabled(DRM_UT_KMS))
2685		return;
2686
2687	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2688					    new_crtc_state, i) {
2689		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2690
2691		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2692		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2693
2694		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2695			enum plane_id plane_id = plane->id;
2696			const struct skl_ddb_entry *old, *new;
2697
2698			old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2699			new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2700
2701			if (skl_ddb_entry_equal(old, new))
2702				continue;
2703
2704			drm_dbg_kms(&i915->drm,
2705				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2706				    plane->base.base.id, plane->base.name,
2707				    old->start, old->end, new->start, new->end,
2708				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2709		}
2710
2711		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2712			enum plane_id plane_id = plane->id;
2713			const struct skl_plane_wm *old_wm, *new_wm;
2714
2715			old_wm = &old_pipe_wm->planes[plane_id];
2716			new_wm = &new_pipe_wm->planes[plane_id];
2717
2718			if (skl_plane_wm_equals(i915, old_wm, new_wm))
2719				continue;
2720
2721			drm_dbg_kms(&i915->drm,
2722				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2723				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2724				    plane->base.base.id, plane->base.name,
2725				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2726				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2727				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2728				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2729				    enast(old_wm->trans_wm.enable),
2730				    enast(old_wm->sagv.wm0.enable),
2731				    enast(old_wm->sagv.trans_wm.enable),
2732				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2733				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2734				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2735				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2736				    enast(new_wm->trans_wm.enable),
2737				    enast(new_wm->sagv.wm0.enable),
2738				    enast(new_wm->sagv.trans_wm.enable));
2739
2740			drm_dbg_kms(&i915->drm,
2741				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2742				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2743				    plane->base.base.id, plane->base.name,
2744				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2745				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2746				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2747				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2748				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2749				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2750				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2751				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2752				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2753				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2754				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2755				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2756				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2757				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2758				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2759				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2760				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2761				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2762				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2763				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2764				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2765				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2766
2767			drm_dbg_kms(&i915->drm,
2768				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2769				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2770				    plane->base.base.id, plane->base.name,
2771				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2772				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2773				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2774				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2775				    old_wm->trans_wm.blocks,
2776				    old_wm->sagv.wm0.blocks,
2777				    old_wm->sagv.trans_wm.blocks,
2778				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2779				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2780				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2781				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2782				    new_wm->trans_wm.blocks,
2783				    new_wm->sagv.wm0.blocks,
2784				    new_wm->sagv.trans_wm.blocks);
2785
2786			drm_dbg_kms(&i915->drm,
2787				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2788				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2789				    plane->base.base.id, plane->base.name,
2790				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2791				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2792				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2793				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2794				    old_wm->trans_wm.min_ddb_alloc,
2795				    old_wm->sagv.wm0.min_ddb_alloc,
2796				    old_wm->sagv.trans_wm.min_ddb_alloc,
2797				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2798				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2799				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2800				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2801				    new_wm->trans_wm.min_ddb_alloc,
2802				    new_wm->sagv.wm0.min_ddb_alloc,
2803				    new_wm->sagv.trans_wm.min_ddb_alloc);
2804		}
2805	}
2806}
2807
2808static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2809					 const struct skl_pipe_wm *old_pipe_wm,
2810					 const struct skl_pipe_wm *new_pipe_wm)
2811{
2812	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2813	int level;
2814
2815	for (level = 0; level < i915->display.wm.num_levels; level++) {
2816		/*
2817		 * We don't check uv_wm as the hardware doesn't actually
2818		 * use it. It only gets used for calculating the required
2819		 * ddb allocation.
2820		 */
2821		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2822					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2823			return false;
2824	}
2825
2826	if (HAS_HW_SAGV_WM(i915)) {
2827		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2828		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2829
2830		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2831		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2832			return false;
2833	}
2834
2835	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2836				   skl_plane_trans_wm(new_pipe_wm, plane->id));
2837}
2838
2839/*
2840 * To make sure the cursor watermark registers are always consistent
2841 * with our computed state the following scenario needs special
2842 * treatment:
2843 *
2844 * 1. enable cursor
2845 * 2. move cursor entirely offscreen
2846 * 3. disable cursor
2847 *
2848 * Step 2. does call .disable_plane() but does not zero the watermarks
2849 * (since we consider an offscreen cursor still active for the purposes
2850 * of watermarks). Step 3. would not normally call .disable_plane()
2851 * because the actual plane visibility isn't changing, and we don't
2852 * deallocate the cursor ddb until the pipe gets disabled. So we must
2853 * force step 3. to call .disable_plane() to update the watermark
2854 * registers properly.
2855 *
2856 * Other planes do not suffer from this issues as their watermarks are
2857 * calculated based on the actual plane visibility. The only time this
2858 * can trigger for the other planes is during the initial readout as the
2859 * default value of the watermarks registers is not zero.
2860 */
2861static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2862				      struct intel_crtc *crtc)
2863{
2864	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2865	const struct intel_crtc_state *old_crtc_state =
2866		intel_atomic_get_old_crtc_state(state, crtc);
2867	struct intel_crtc_state *new_crtc_state =
2868		intel_atomic_get_new_crtc_state(state, crtc);
2869	struct intel_plane *plane;
2870
2871	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2872		struct intel_plane_state *plane_state;
2873		enum plane_id plane_id = plane->id;
2874
2875		/*
2876		 * Force a full wm update for every plane on modeset.
2877		 * Required because the reset value of the wm registers
2878		 * is non-zero, whereas we want all disabled planes to
2879		 * have zero watermarks. So if we turn off the relevant
2880		 * power well the hardware state will go out of sync
2881		 * with the software state.
2882		 */
2883		if (!intel_crtc_needs_modeset(new_crtc_state) &&
2884		    skl_plane_selected_wm_equals(plane,
2885						 &old_crtc_state->wm.skl.optimal,
2886						 &new_crtc_state->wm.skl.optimal))
2887			continue;
2888
2889		plane_state = intel_atomic_get_plane_state(state, plane);
2890		if (IS_ERR(plane_state))
2891			return PTR_ERR(plane_state);
2892
2893		new_crtc_state->update_planes |= BIT(plane_id);
2894		new_crtc_state->async_flip_planes = 0;
2895		new_crtc_state->do_async_flip = false;
2896	}
2897
2898	return 0;
2899}
2900
2901static int
2902skl_compute_wm(struct intel_atomic_state *state)
2903{
2904	struct intel_crtc *crtc;
2905	struct intel_crtc_state __maybe_unused *new_crtc_state;
2906	int ret, i;
2907
2908	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2909		ret = skl_build_pipe_wm(state, crtc);
2910		if (ret)
2911			return ret;
2912	}
2913
2914	ret = skl_compute_ddb(state);
2915	if (ret)
2916		return ret;
2917
2918	ret = intel_compute_sagv_mask(state);
2919	if (ret)
2920		return ret;
2921
2922	/*
2923	 * skl_compute_ddb() will have adjusted the final watermarks
2924	 * based on how much ddb is available. Now we can actually
2925	 * check if the final watermarks changed.
2926	 */
2927	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2928		ret = skl_wm_add_affected_planes(state, crtc);
2929		if (ret)
2930			return ret;
2931	}
2932
2933	skl_print_wm_changes(state);
2934
2935	return 0;
2936}
2937
2938static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2939{
2940	level->enable = val & PLANE_WM_EN;
2941	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2942	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2943	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2944}
2945
2946static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2947				     struct skl_pipe_wm *out)
2948{
2949	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2950	enum pipe pipe = crtc->pipe;
2951	enum plane_id plane_id;
2952	int level;
2953	u32 val;
2954
2955	for_each_plane_id_on_crtc(crtc, plane_id) {
2956		struct skl_plane_wm *wm = &out->planes[plane_id];
2957
2958		for (level = 0; level < i915->display.wm.num_levels; level++) {
2959			if (plane_id != PLANE_CURSOR)
2960				val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2961			else
2962				val = intel_de_read(i915, CUR_WM(pipe, level));
2963
2964			skl_wm_level_from_reg_val(val, &wm->wm[level]);
2965		}
2966
2967		if (plane_id != PLANE_CURSOR)
2968			val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2969		else
2970			val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2971
2972		skl_wm_level_from_reg_val(val, &wm->trans_wm);
2973
2974		if (HAS_HW_SAGV_WM(i915)) {
2975			if (plane_id != PLANE_CURSOR)
2976				val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2977			else
2978				val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2979
2980			skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2981
2982			if (plane_id != PLANE_CURSOR)
2983				val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2984			else
2985				val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2986
2987			skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2988		} else if (DISPLAY_VER(i915) >= 12) {
2989			wm->sagv.wm0 = wm->wm[0];
2990			wm->sagv.trans_wm = wm->trans_wm;
2991		}
2992	}
2993}
2994
2995static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2996{
2997	struct intel_dbuf_state *dbuf_state =
2998		to_intel_dbuf_state(i915->display.dbuf.obj.state);
2999	struct intel_crtc *crtc;
3000
3001	if (HAS_MBUS_JOINING(i915))
3002		dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
3003
3004	for_each_intel_crtc(&i915->drm, crtc) {
3005		struct intel_crtc_state *crtc_state =
3006			to_intel_crtc_state(crtc->base.state);
3007		enum pipe pipe = crtc->pipe;
3008		unsigned int mbus_offset;
3009		enum plane_id plane_id;
3010		u8 slices;
3011
3012		memset(&crtc_state->wm.skl.optimal, 0,
3013		       sizeof(crtc_state->wm.skl.optimal));
3014		if (crtc_state->hw.active)
3015			skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3016		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3017
3018		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3019
3020		for_each_plane_id_on_crtc(crtc, plane_id) {
3021			struct skl_ddb_entry *ddb =
3022				&crtc_state->wm.skl.plane_ddb[plane_id];
3023			struct skl_ddb_entry *ddb_y =
3024				&crtc_state->wm.skl.plane_ddb_y[plane_id];
3025
3026			if (!crtc_state->hw.active)
3027				continue;
3028
3029			skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3030						   plane_id, ddb, ddb_y);
3031
3032			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3033			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3034		}
3035
3036		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3037
3038		/*
3039		 * Used for checking overlaps, so we need absolute
3040		 * offsets instead of MBUS relative offsets.
3041		 */
3042		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3043						 dbuf_state->joined_mbus);
3044		mbus_offset = mbus_ddb_offset(i915, slices);
3045		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3046		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3047
3048		/* The slices actually used by the planes on the pipe */
3049		dbuf_state->slices[pipe] =
3050			skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3051
3052		drm_dbg_kms(&i915->drm,
3053			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3054			    crtc->base.base.id, crtc->base.name,
3055			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3056			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3057			    str_yes_no(dbuf_state->joined_mbus));
3058	}
3059
3060	dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3061}
3062
3063static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3064{
3065	const struct intel_dbuf_state *dbuf_state =
3066		to_intel_dbuf_state(i915->display.dbuf.obj.state);
3067	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3068	struct intel_crtc *crtc;
3069
3070	for_each_intel_crtc(&i915->drm, crtc) {
3071		const struct intel_crtc_state *crtc_state =
3072			to_intel_crtc_state(crtc->base.state);
3073
3074		entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3075	}
3076
3077	for_each_intel_crtc(&i915->drm, crtc) {
3078		const struct intel_crtc_state *crtc_state =
3079			to_intel_crtc_state(crtc->base.state);
3080		u8 slices;
3081
3082		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3083						 dbuf_state->joined_mbus);
3084		if (dbuf_state->slices[crtc->pipe] & ~slices)
3085			return true;
3086
3087		if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3088						I915_MAX_PIPES, crtc->pipe))
3089			return true;
3090	}
3091
3092	return false;
3093}
3094
3095static void skl_wm_sanitize(struct drm_i915_private *i915)
3096{
3097	struct intel_crtc *crtc;
3098
3099	/*
3100	 * On TGL/RKL (at least) the BIOS likes to assign the planes
3101	 * to the wrong DBUF slices. This will cause an infinite loop
3102	 * in skl_commit_modeset_enables() as it can't find a way to
3103	 * transition between the old bogus DBUF layout to the new
3104	 * proper DBUF layout without DBUF allocation overlaps between
3105	 * the planes (which cannot be allowed or else the hardware
3106	 * may hang). If we detect a bogus DBUF layout just turn off
3107	 * all the planes so that skl_commit_modeset_enables() can
3108	 * simply ignore them.
3109	 */
3110	if (!skl_dbuf_is_misconfigured(i915))
3111		return;
3112
3113	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3114
3115	for_each_intel_crtc(&i915->drm, crtc) {
3116		struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3117		const struct intel_plane_state *plane_state =
3118			to_intel_plane_state(plane->base.state);
3119		struct intel_crtc_state *crtc_state =
3120			to_intel_crtc_state(crtc->base.state);
3121
3122		if (plane_state->uapi.visible)
3123			intel_plane_disable_noatomic(crtc, plane);
3124
3125		drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3126
3127		memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3128	}
3129}
3130
3131static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3132{
3133	skl_wm_get_hw_state(i915);
3134	skl_wm_sanitize(i915);
3135}
3136
3137void intel_wm_state_verify(struct intel_atomic_state *state,
3138			   struct intel_crtc *crtc)
3139{
3140	struct drm_i915_private *i915 = to_i915(state->base.dev);
3141	const struct intel_crtc_state *new_crtc_state =
3142		intel_atomic_get_new_crtc_state(state, crtc);
3143	struct skl_hw_state {
3144		struct skl_ddb_entry ddb[I915_MAX_PLANES];
3145		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3146		struct skl_pipe_wm wm;
3147	} *hw;
3148	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3149	struct intel_plane *plane;
3150	u8 hw_enabled_slices;
3151	int level;
3152
3153	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3154		return;
3155
3156	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3157	if (!hw)
3158		return;
3159
3160	skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3161
3162	skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3163
3164	hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3165
3166	if (DISPLAY_VER(i915) >= 11 &&
3167	    hw_enabled_slices != i915->display.dbuf.enabled_slices)
3168		drm_err(&i915->drm,
3169			"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3170			i915->display.dbuf.enabled_slices,
3171			hw_enabled_slices);
3172
3173	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3174		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3175		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3176
3177		/* Watermarks */
3178		for (level = 0; level < i915->display.wm.num_levels; level++) {
3179			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3180			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3181
3182			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3183				continue;
3184
3185			drm_err(&i915->drm,
3186				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3187				plane->base.base.id, plane->base.name, level,
3188				sw_wm_level->enable,
3189				sw_wm_level->blocks,
3190				sw_wm_level->lines,
3191				hw_wm_level->enable,
3192				hw_wm_level->blocks,
3193				hw_wm_level->lines);
3194		}
3195
3196		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3197		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3198
3199		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3200			drm_err(&i915->drm,
3201				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3202				plane->base.base.id, plane->base.name,
3203				sw_wm_level->enable,
3204				sw_wm_level->blocks,
3205				sw_wm_level->lines,
3206				hw_wm_level->enable,
3207				hw_wm_level->blocks,
3208				hw_wm_level->lines);
3209		}
3210
3211		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3212		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3213
3214		if (HAS_HW_SAGV_WM(i915) &&
3215		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3216			drm_err(&i915->drm,
3217				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3218				plane->base.base.id, plane->base.name,
3219				sw_wm_level->enable,
3220				sw_wm_level->blocks,
3221				sw_wm_level->lines,
3222				hw_wm_level->enable,
3223				hw_wm_level->blocks,
3224				hw_wm_level->lines);
3225		}
3226
3227		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3228		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3229
3230		if (HAS_HW_SAGV_WM(i915) &&
3231		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3232			drm_err(&i915->drm,
3233				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3234				plane->base.base.id, plane->base.name,
3235				sw_wm_level->enable,
3236				sw_wm_level->blocks,
3237				sw_wm_level->lines,
3238				hw_wm_level->enable,
3239				hw_wm_level->blocks,
3240				hw_wm_level->lines);
3241		}
3242
3243		/* DDB */
3244		hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3245		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3246
3247		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3248			drm_err(&i915->drm,
3249				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3250				plane->base.base.id, plane->base.name,
3251				sw_ddb_entry->start, sw_ddb_entry->end,
3252				hw_ddb_entry->start, hw_ddb_entry->end);
3253		}
3254	}
3255
3256	kfree(hw);
3257}
3258
3259bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3260{
3261	return i915->display.wm.ipc_enabled;
3262}
3263
3264void skl_watermark_ipc_update(struct drm_i915_private *i915)
3265{
3266	if (!HAS_IPC(i915))
3267		return;
3268
3269	intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3270		     skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3271}
3272
3273static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3274{
3275	/* Display WA #0477 WaDisableIPC: skl */
3276	if (IS_SKYLAKE(i915))
3277		return false;
3278
3279	/* Display WA #1141: SKL:all KBL:all CFL */
3280	if (IS_KABYLAKE(i915) ||
3281	    IS_COFFEELAKE(i915) ||
3282	    IS_COMETLAKE(i915))
3283		return i915->dram_info.symmetric_memory;
3284
3285	return true;
3286}
3287
3288void skl_watermark_ipc_init(struct drm_i915_private *i915)
3289{
3290	if (!HAS_IPC(i915))
3291		return;
3292
3293	i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3294
3295	skl_watermark_ipc_update(i915);
3296}
3297
3298static void
3299adjust_wm_latency(struct drm_i915_private *i915,
3300		  u16 wm[], int num_levels, int read_latency)
3301{
3302	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3303	int i, level;
3304
3305	/*
3306	 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3307	 * need to be disabled. We make sure to sanitize the values out
3308	 * of the punit to satisfy this requirement.
3309	 */
3310	for (level = 1; level < num_levels; level++) {
3311		if (wm[level] == 0) {
3312			for (i = level + 1; i < num_levels; i++)
3313				wm[i] = 0;
3314
3315			num_levels = level;
3316			break;
3317		}
3318	}
3319
3320	/*
3321	 * WaWmMemoryReadLatency
3322	 *
3323	 * punit doesn't take into account the read latency so we need
3324	 * to add proper adjustement to each valid level we retrieve
3325	 * from the punit when level 0 response data is 0us.
3326	 */
3327	if (wm[0] == 0) {
3328		for (level = 0; level < num_levels; level++)
3329			wm[level] += read_latency;
3330	}
3331
3332	/*
3333	 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3334	 * If we could not get dimm info enable this WA to prevent from
3335	 * any underrun. If not able to get Dimm info assume 16GB dimm
3336	 * to avoid any underrun.
3337	 */
3338	if (wm_lv_0_adjust_needed)
3339		wm[0] += 1;
3340}
3341
3342static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3343{
3344	int num_levels = i915->display.wm.num_levels;
3345	u32 val;
3346
3347	val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3348	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3349	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3350
3351	val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3352	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3353	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3354
3355	val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3356	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3357	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3358
3359	adjust_wm_latency(i915, wm, num_levels, 6);
3360}
3361
3362static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3363{
3364	int num_levels = i915->display.wm.num_levels;
3365	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3366	int mult = IS_DG2(i915) ? 2 : 1;
3367	u32 val;
3368	int ret;
3369
3370	/* read the first set of memory latencies[0:3] */
3371	val = 0; /* data0 to be programmed to 0 for first set */
3372	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3373	if (ret) {
3374		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3375		return;
3376	}
3377
3378	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3379	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3380	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3381	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3382
3383	/* read the second set of memory latencies[4:7] */
3384	val = 1; /* data0 to be programmed to 1 for second set */
3385	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3386	if (ret) {
3387		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3388		return;
3389	}
3390
3391	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3392	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3393	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3394	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3395
3396	adjust_wm_latency(i915, wm, num_levels, read_latency);
3397}
3398
3399static void skl_setup_wm_latency(struct drm_i915_private *i915)
3400{
3401	if (HAS_HW_SAGV_WM(i915))
3402		i915->display.wm.num_levels = 6;
3403	else
3404		i915->display.wm.num_levels = 8;
3405
3406	if (DISPLAY_VER(i915) >= 14)
3407		mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3408	else
3409		skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3410
3411	intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3412}
3413
3414static const struct intel_wm_funcs skl_wm_funcs = {
3415	.compute_global_watermarks = skl_compute_wm,
3416	.get_hw_state = skl_wm_get_hw_state_and_sanitize,
3417};
3418
3419void skl_wm_init(struct drm_i915_private *i915)
3420{
3421	intel_sagv_init(i915);
3422
3423	skl_setup_wm_latency(i915);
3424
3425	i915->display.funcs.wm = &skl_wm_funcs;
3426}
3427
3428static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3429{
3430	struct intel_dbuf_state *dbuf_state;
3431
3432	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3433	if (!dbuf_state)
3434		return NULL;
3435
3436	return &dbuf_state->base;
3437}
3438
3439static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3440				     struct intel_global_state *state)
3441{
3442	kfree(state);
3443}
3444
3445static const struct intel_global_state_funcs intel_dbuf_funcs = {
3446	.atomic_duplicate_state = intel_dbuf_duplicate_state,
3447	.atomic_destroy_state = intel_dbuf_destroy_state,
3448};
3449
3450struct intel_dbuf_state *
3451intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3452{
3453	struct drm_i915_private *i915 = to_i915(state->base.dev);
3454	struct intel_global_state *dbuf_state;
3455
3456	dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3457	if (IS_ERR(dbuf_state))
3458		return ERR_CAST(dbuf_state);
3459
3460	return to_intel_dbuf_state(dbuf_state);
3461}
3462
3463int intel_dbuf_init(struct drm_i915_private *i915)
3464{
3465	struct intel_dbuf_state *dbuf_state;
3466
3467	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3468	if (!dbuf_state)
3469		return -ENOMEM;
3470
3471	intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3472				     &dbuf_state->base, &intel_dbuf_funcs);
3473
3474	return 0;
3475}
3476
3477/*
3478 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3479 * update the request state of all DBUS slices.
3480 */
3481static void update_mbus_pre_enable(struct intel_atomic_state *state)
3482{
3483	struct drm_i915_private *i915 = to_i915(state->base.dev);
3484	u32 mbus_ctl, dbuf_min_tracker_val;
3485	enum dbuf_slice slice;
3486	const struct intel_dbuf_state *dbuf_state =
3487		intel_atomic_get_new_dbuf_state(state);
3488
3489	if (!HAS_MBUS_JOINING(i915))
3490		return;
3491
3492	/*
3493	 * TODO: Implement vblank synchronized MBUS joining changes.
3494	 * Must be properly coordinated with dbuf reprogramming.
3495	 */
3496	if (dbuf_state->joined_mbus) {
3497		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3498			MBUS_JOIN_PIPE_SELECT_NONE;
3499		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3500	} else {
3501		mbus_ctl = MBUS_HASHING_MODE_2x2 |
3502			MBUS_JOIN_PIPE_SELECT_NONE;
3503		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3504	}
3505
3506	intel_de_rmw(i915, MBUS_CTL,
3507		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3508		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3509
3510	for_each_dbuf_slice(i915, slice)
3511		intel_de_rmw(i915, DBUF_CTL_S(slice),
3512			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3513			     dbuf_min_tracker_val);
3514}
3515
3516void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3517{
3518	struct drm_i915_private *i915 = to_i915(state->base.dev);
3519	const struct intel_dbuf_state *new_dbuf_state =
3520		intel_atomic_get_new_dbuf_state(state);
3521	const struct intel_dbuf_state *old_dbuf_state =
3522		intel_atomic_get_old_dbuf_state(state);
3523
3524	if (!new_dbuf_state ||
3525	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3526	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3527		return;
3528
3529	WARN_ON(!new_dbuf_state->base.changed);
3530
3531	update_mbus_pre_enable(state);
3532	gen9_dbuf_slices_update(i915,
3533				old_dbuf_state->enabled_slices |
3534				new_dbuf_state->enabled_slices);
3535}
3536
3537void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3538{
3539	struct drm_i915_private *i915 = to_i915(state->base.dev);
3540	const struct intel_dbuf_state *new_dbuf_state =
3541		intel_atomic_get_new_dbuf_state(state);
3542	const struct intel_dbuf_state *old_dbuf_state =
3543		intel_atomic_get_old_dbuf_state(state);
3544
3545	if (!new_dbuf_state ||
3546	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3547	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3548		return;
3549
3550	WARN_ON(!new_dbuf_state->base.changed);
3551
3552	gen9_dbuf_slices_update(i915,
3553				new_dbuf_state->enabled_slices);
3554}
3555
3556static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3557{
3558	switch (pipe) {
3559	case PIPE_A:
3560		return !(active_pipes & BIT(PIPE_D));
3561	case PIPE_D:
3562		return !(active_pipes & BIT(PIPE_A));
3563	case PIPE_B:
3564		return !(active_pipes & BIT(PIPE_C));
3565	case PIPE_C:
3566		return !(active_pipes & BIT(PIPE_B));
3567	default: /* to suppress compiler warning */
3568		MISSING_CASE(pipe);
3569		break;
3570	}
3571
3572	return false;
3573}
3574
3575void intel_mbus_dbox_update(struct intel_atomic_state *state)
3576{
3577	struct drm_i915_private *i915 = to_i915(state->base.dev);
3578	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3579	const struct intel_crtc_state *new_crtc_state;
3580	const struct intel_crtc *crtc;
3581	u32 val = 0;
3582	int i;
3583
3584	if (DISPLAY_VER(i915) < 11)
3585		return;
3586
3587	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3588	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3589	if (!new_dbuf_state ||
3590	    (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3591	     new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3592		return;
3593
3594	if (DISPLAY_VER(i915) >= 14)
3595		val |= MBUS_DBOX_I_CREDIT(2);
3596
3597	if (DISPLAY_VER(i915) >= 12) {
3598		val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3599		val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3600		val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3601	}
3602
3603	if (DISPLAY_VER(i915) >= 14)
3604		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3605						     MBUS_DBOX_A_CREDIT(8);
3606	else if (IS_ALDERLAKE_P(i915))
3607		/* Wa_22010947358:adl-p */
3608		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3609						     MBUS_DBOX_A_CREDIT(4);
3610	else
3611		val |= MBUS_DBOX_A_CREDIT(2);
3612
3613	if (DISPLAY_VER(i915) >= 14) {
3614		val |= MBUS_DBOX_B_CREDIT(0xA);
3615	} else if (IS_ALDERLAKE_P(i915)) {
3616		val |= MBUS_DBOX_BW_CREDIT(2);
3617		val |= MBUS_DBOX_B_CREDIT(8);
3618	} else if (DISPLAY_VER(i915) >= 12) {
3619		val |= MBUS_DBOX_BW_CREDIT(2);
3620		val |= MBUS_DBOX_B_CREDIT(12);
3621	} else {
3622		val |= MBUS_DBOX_BW_CREDIT(1);
3623		val |= MBUS_DBOX_B_CREDIT(8);
3624	}
3625
3626	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3627		u32 pipe_val = val;
3628
3629		if (!new_crtc_state->hw.active)
3630			continue;
3631
3632		if (DISPLAY_VER(i915) >= 14) {
3633			if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3634							      new_dbuf_state->active_pipes))
3635				pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3636			else
3637				pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3638		}
3639
3640		intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3641	}
3642}
3643
3644static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3645{
3646	struct drm_i915_private *i915 = m->private;
3647
3648	seq_printf(m, "Isochronous Priority Control: %s\n",
3649		   str_yes_no(skl_watermark_ipc_enabled(i915)));
3650	return 0;
3651}
3652
3653static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3654{
3655	struct drm_i915_private *i915 = inode->i_private;
3656
3657	return single_open(file, skl_watermark_ipc_status_show, i915);
3658}
3659
3660static ssize_t skl_watermark_ipc_status_write(struct file *file,
3661					      const char __user *ubuf,
3662					      size_t len, loff_t *offp)
3663{
3664	struct seq_file *m = file->private_data;
3665	struct drm_i915_private *i915 = m->private;
3666	intel_wakeref_t wakeref;
3667	bool enable;
3668	int ret;
3669
3670	ret = kstrtobool_from_user(ubuf, len, &enable);
3671	if (ret < 0)
3672		return ret;
3673
3674	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3675		if (!skl_watermark_ipc_enabled(i915) && enable)
3676			drm_info(&i915->drm,
3677				 "Enabling IPC: WM will be proper only after next commit\n");
3678		i915->display.wm.ipc_enabled = enable;
3679		skl_watermark_ipc_update(i915);
3680	}
3681
3682	return len;
3683}
3684
3685static const struct file_operations skl_watermark_ipc_status_fops = {
3686	.owner = THIS_MODULE,
3687	.open = skl_watermark_ipc_status_open,
3688	.read = seq_read,
3689	.llseek = seq_lseek,
3690	.release = single_release,
3691	.write = skl_watermark_ipc_status_write
3692};
3693
3694static int intel_sagv_status_show(struct seq_file *m, void *unused)
3695{
3696	struct drm_i915_private *i915 = m->private;
3697	static const char * const sagv_status[] = {
3698		[I915_SAGV_UNKNOWN] = "unknown",
3699		[I915_SAGV_DISABLED] = "disabled",
3700		[I915_SAGV_ENABLED] = "enabled",
3701		[I915_SAGV_NOT_CONTROLLED] = "not controlled",
3702	};
3703
3704	seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3705	seq_printf(m, "SAGV modparam: %s\n",
3706		   str_enabled_disabled(i915->display.params.enable_sagv));
3707	seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3708	seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3709
3710	return 0;
3711}
3712
3713DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3714
3715void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3716{
3717	struct drm_minor *minor = i915->drm.primary;
3718
3719	if (HAS_IPC(i915))
3720		debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3721				    &skl_watermark_ipc_status_fops);
3722
3723	if (HAS_SAGV(i915))
3724		debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3725				    &intel_sagv_status_fops);
3726}
3727
3728unsigned int skl_watermark_max_latency(struct drm_i915_private *i915)
3729{
3730	int level;
3731
3732	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
3733		unsigned int latency = skl_wm_latency(i915, level, NULL);
3734
3735		if (latency)
3736			return latency;
3737	}
3738
3739	return 0;
3740}