1/* SPDX-License-Identifier: MIT */
   2/*
   3 * Copyright © 2019 Intel Corporation
   4 */
   5
   6#include <linux/string_helpers.h>
   7
   8#include "i915_drv.h"
   9#include "i915_irq.h"
  10#include "i915_reg.h"
  11#include "intel_backlight_regs.h"
  12#include "intel_cdclk.h"
  13#include "intel_clock_gating.h"
  14#include "intel_combo_phy.h"
  15#include "intel_de.h"
  16#include "intel_display_power.h"
  17#include "intel_display_power_map.h"
  18#include "intel_display_power_well.h"
  19#include "intel_display_types.h"
  20#include "intel_dmc.h"
  21#include "intel_mchbar_regs.h"
  22#include "intel_pch_refclk.h"
  23#include "intel_pcode.h"
  24#include "intel_pmdemand.h"
  25#include "intel_pps_regs.h"
  26#include "intel_snps_phy.h"
  27#include "skl_watermark.h"
  28#include "skl_watermark_regs.h"
  29#include "vlv_sideband.h"
  30
  31#define for_each_power_domain_well(__dev_priv, __power_well, __domain)	\
  32	for_each_power_well(__dev_priv, __power_well)				\
  33		for_each_if(test_bit((__domain), (__power_well)->domains.bits))
  34
  35#define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain) \
  36	for_each_power_well_reverse(__dev_priv, __power_well)		        \
  37		for_each_if(test_bit((__domain), (__power_well)->domains.bits))
  38
  39static const char *
  40intel_display_power_domain_str(enum intel_display_power_domain domain)
  41{
  42	switch (domain) {
  43	case POWER_DOMAIN_DISPLAY_CORE:
  44		return "DISPLAY_CORE";
  45	case POWER_DOMAIN_PIPE_A:
  46		return "PIPE_A";
  47	case POWER_DOMAIN_PIPE_B:
  48		return "PIPE_B";
  49	case POWER_DOMAIN_PIPE_C:
  50		return "PIPE_C";
  51	case POWER_DOMAIN_PIPE_D:
  52		return "PIPE_D";
  53	case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
  54		return "PIPE_PANEL_FITTER_A";
  55	case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
  56		return "PIPE_PANEL_FITTER_B";
  57	case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
  58		return "PIPE_PANEL_FITTER_C";
  59	case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
  60		return "PIPE_PANEL_FITTER_D";
  61	case POWER_DOMAIN_TRANSCODER_A:
  62		return "TRANSCODER_A";
  63	case POWER_DOMAIN_TRANSCODER_B:
  64		return "TRANSCODER_B";
  65	case POWER_DOMAIN_TRANSCODER_C:
  66		return "TRANSCODER_C";
  67	case POWER_DOMAIN_TRANSCODER_D:
  68		return "TRANSCODER_D";
  69	case POWER_DOMAIN_TRANSCODER_EDP:
  70		return "TRANSCODER_EDP";
  71	case POWER_DOMAIN_TRANSCODER_DSI_A:
  72		return "TRANSCODER_DSI_A";
  73	case POWER_DOMAIN_TRANSCODER_DSI_C:
  74		return "TRANSCODER_DSI_C";
  75	case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
  76		return "TRANSCODER_VDSC_PW2";
  77	case POWER_DOMAIN_PORT_DDI_LANES_A:
  78		return "PORT_DDI_LANES_A";
  79	case POWER_DOMAIN_PORT_DDI_LANES_B:
  80		return "PORT_DDI_LANES_B";
  81	case POWER_DOMAIN_PORT_DDI_LANES_C:
  82		return "PORT_DDI_LANES_C";
  83	case POWER_DOMAIN_PORT_DDI_LANES_D:
  84		return "PORT_DDI_LANES_D";
  85	case POWER_DOMAIN_PORT_DDI_LANES_E:
  86		return "PORT_DDI_LANES_E";
  87	case POWER_DOMAIN_PORT_DDI_LANES_F:
  88		return "PORT_DDI_LANES_F";
  89	case POWER_DOMAIN_PORT_DDI_LANES_TC1:
  90		return "PORT_DDI_LANES_TC1";
  91	case POWER_DOMAIN_PORT_DDI_LANES_TC2:
  92		return "PORT_DDI_LANES_TC2";
  93	case POWER_DOMAIN_PORT_DDI_LANES_TC3:
  94		return "PORT_DDI_LANES_TC3";
  95	case POWER_DOMAIN_PORT_DDI_LANES_TC4:
  96		return "PORT_DDI_LANES_TC4";
  97	case POWER_DOMAIN_PORT_DDI_LANES_TC5:
  98		return "PORT_DDI_LANES_TC5";
  99	case POWER_DOMAIN_PORT_DDI_LANES_TC6:
 100		return "PORT_DDI_LANES_TC6";
 101	case POWER_DOMAIN_PORT_DDI_IO_A:
 102		return "PORT_DDI_IO_A";
 103	case POWER_DOMAIN_PORT_DDI_IO_B:
 104		return "PORT_DDI_IO_B";
 105	case POWER_DOMAIN_PORT_DDI_IO_C:
 106		return "PORT_DDI_IO_C";
 107	case POWER_DOMAIN_PORT_DDI_IO_D:
 108		return "PORT_DDI_IO_D";
 109	case POWER_DOMAIN_PORT_DDI_IO_E:
 110		return "PORT_DDI_IO_E";
 111	case POWER_DOMAIN_PORT_DDI_IO_F:
 112		return "PORT_DDI_IO_F";
 113	case POWER_DOMAIN_PORT_DDI_IO_TC1:
 114		return "PORT_DDI_IO_TC1";
 115	case POWER_DOMAIN_PORT_DDI_IO_TC2:
 116		return "PORT_DDI_IO_TC2";
 117	case POWER_DOMAIN_PORT_DDI_IO_TC3:
 118		return "PORT_DDI_IO_TC3";
 119	case POWER_DOMAIN_PORT_DDI_IO_TC4:
 120		return "PORT_DDI_IO_TC4";
 121	case POWER_DOMAIN_PORT_DDI_IO_TC5:
 122		return "PORT_DDI_IO_TC5";
 123	case POWER_DOMAIN_PORT_DDI_IO_TC6:
 124		return "PORT_DDI_IO_TC6";
 125	case POWER_DOMAIN_PORT_DSI:
 126		return "PORT_DSI";
 127	case POWER_DOMAIN_PORT_CRT:
 128		return "PORT_CRT";
 129	case POWER_DOMAIN_PORT_OTHER:
 130		return "PORT_OTHER";
 131	case POWER_DOMAIN_VGA:
 132		return "VGA";
 133	case POWER_DOMAIN_AUDIO_MMIO:
 134		return "AUDIO_MMIO";
 135	case POWER_DOMAIN_AUDIO_PLAYBACK:
 136		return "AUDIO_PLAYBACK";
 137	case POWER_DOMAIN_AUX_IO_A:
 138		return "AUX_IO_A";
 139	case POWER_DOMAIN_AUX_IO_B:
 140		return "AUX_IO_B";
 141	case POWER_DOMAIN_AUX_IO_C:
 142		return "AUX_IO_C";
 143	case POWER_DOMAIN_AUX_IO_D:
 144		return "AUX_IO_D";
 145	case POWER_DOMAIN_AUX_IO_E:
 146		return "AUX_IO_E";
 147	case POWER_DOMAIN_AUX_IO_F:
 148		return "AUX_IO_F";
 149	case POWER_DOMAIN_AUX_A:
 150		return "AUX_A";
 151	case POWER_DOMAIN_AUX_B:
 152		return "AUX_B";
 153	case POWER_DOMAIN_AUX_C:
 154		return "AUX_C";
 155	case POWER_DOMAIN_AUX_D:
 156		return "AUX_D";
 157	case POWER_DOMAIN_AUX_E:
 158		return "AUX_E";
 159	case POWER_DOMAIN_AUX_F:
 160		return "AUX_F";
 161	case POWER_DOMAIN_AUX_USBC1:
 162		return "AUX_USBC1";
 163	case POWER_DOMAIN_AUX_USBC2:
 164		return "AUX_USBC2";
 165	case POWER_DOMAIN_AUX_USBC3:
 166		return "AUX_USBC3";
 167	case POWER_DOMAIN_AUX_USBC4:
 168		return "AUX_USBC4";
 169	case POWER_DOMAIN_AUX_USBC5:
 170		return "AUX_USBC5";
 171	case POWER_DOMAIN_AUX_USBC6:
 172		return "AUX_USBC6";
 173	case POWER_DOMAIN_AUX_TBT1:
 174		return "AUX_TBT1";
 175	case POWER_DOMAIN_AUX_TBT2:
 176		return "AUX_TBT2";
 177	case POWER_DOMAIN_AUX_TBT3:
 178		return "AUX_TBT3";
 179	case POWER_DOMAIN_AUX_TBT4:
 180		return "AUX_TBT4";
 181	case POWER_DOMAIN_AUX_TBT5:
 182		return "AUX_TBT5";
 183	case POWER_DOMAIN_AUX_TBT6:
 184		return "AUX_TBT6";
 185	case POWER_DOMAIN_GMBUS:
 186		return "GMBUS";
 187	case POWER_DOMAIN_INIT:
 188		return "INIT";
 189	case POWER_DOMAIN_GT_IRQ:
 190		return "GT_IRQ";
 191	case POWER_DOMAIN_DC_OFF:
 192		return "DC_OFF";
 193	case POWER_DOMAIN_TC_COLD_OFF:
 194		return "TC_COLD_OFF";
 195	default:
 196		MISSING_CASE(domain);
 197		return "?";
 198	}
 199}
 200
 201static bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 202					     enum intel_display_power_domain domain)
 203{
 204	struct i915_power_well *power_well;
 205	bool is_enabled;
 206
 207	if (pm_runtime_suspended(dev_priv->drm.dev))
 208		return false;
 209
 210	is_enabled = true;
 211
 212	for_each_power_domain_well_reverse(dev_priv, power_well, domain) {
 213		if (intel_power_well_is_always_on(power_well))
 214			continue;
 215
 216		if (!intel_power_well_is_enabled_cached(power_well)) {
 217			is_enabled = false;
 218			break;
 219		}
 220	}
 221
 222	return is_enabled;
 223}
 224
 225/**
 226 * intel_display_power_is_enabled - check for a power domain
 227 * @dev_priv: i915 device instance
 228 * @domain: power domain to check
 229 *
 230 * This function can be used to check the hw power domain state. It is mostly
 231 * used in hardware state readout functions. Everywhere else code should rely
 232 * upon explicit power domain reference counting to ensure that the hardware
 233 * block is powered up before accessing it.
 234 *
 235 * Callers must hold the relevant modesetting locks to ensure that concurrent
 236 * threads can't disable the power well while the caller tries to read a few
 237 * registers.
 238 *
 239 * Returns:
 240 * True when the power domain is enabled, false otherwise.
 241 */
 242bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 243				    enum intel_display_power_domain domain)
 244{
 245	struct i915_power_domains *power_domains;
 246	bool ret;
 247
 248	power_domains = &dev_priv->display.power.domains;
 249
 250	mutex_lock(&power_domains->lock);
 251	ret = __intel_display_power_is_enabled(dev_priv, domain);
 252	mutex_unlock(&power_domains->lock);
 253
 254	return ret;
 255}
 256
 257static u32
 258sanitize_target_dc_state(struct drm_i915_private *i915,
 259			 u32 target_dc_state)
 260{
 261	struct i915_power_domains *power_domains = &i915->display.power.domains;
 262	static const u32 states[] = {
 263		DC_STATE_EN_UPTO_DC6,
 264		DC_STATE_EN_UPTO_DC5,
 265		DC_STATE_EN_DC3CO,
 266		DC_STATE_DISABLE,
 267	};
 268	int i;
 269
 270	for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
 271		if (target_dc_state != states[i])
 272			continue;
 273
 274		if (power_domains->allowed_dc_mask & target_dc_state)
 275			break;
 276
 277		target_dc_state = states[i + 1];
 278	}
 279
 280	return target_dc_state;
 281}
 282
 283/**
 284 * intel_display_power_set_target_dc_state - Set target dc state.
 285 * @dev_priv: i915 device
 286 * @state: state which needs to be set as target_dc_state.
 287 *
 288 * This function set the "DC off" power well target_dc_state,
 289 * based upon this target_dc_stste, "DC off" power well will
 290 * enable desired DC state.
 291 */
 292void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 293					     u32 state)
 294{
 295	struct i915_power_well *power_well;
 296	bool dc_off_enabled;
 297	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 298
 299	mutex_lock(&power_domains->lock);
 300	power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
 301
 302	if (drm_WARN_ON(&dev_priv->drm, !power_well))
 303		goto unlock;
 304
 305	state = sanitize_target_dc_state(dev_priv, state);
 306
 307	if (state == power_domains->target_dc_state)
 308		goto unlock;
 309
 310	dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
 311	/*
 312	 * If DC off power well is disabled, need to enable and disable the
 313	 * DC off power well to effect target DC state.
 314	 */
 315	if (!dc_off_enabled)
 316		intel_power_well_enable(dev_priv, power_well);
 317
 318	power_domains->target_dc_state = state;
 319
 320	if (!dc_off_enabled)
 321		intel_power_well_disable(dev_priv, power_well);
 322
 323unlock:
 324	mutex_unlock(&power_domains->lock);
 325}
 326
 327static void __async_put_domains_mask(struct i915_power_domains *power_domains,
 328				     struct intel_power_domain_mask *mask)
 329{
 330	bitmap_or(mask->bits,
 331		  power_domains->async_put_domains[0].bits,
 332		  power_domains->async_put_domains[1].bits,
 333		  POWER_DOMAIN_NUM);
 334}
 335
 336#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 337
 338static bool
 339assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
 340{
 341	struct drm_i915_private *i915 = container_of(power_domains,
 342						     struct drm_i915_private,
 343						     display.power.domains);
 344
 345	return !drm_WARN_ON(&i915->drm,
 346			    bitmap_intersects(power_domains->async_put_domains[0].bits,
 347					      power_domains->async_put_domains[1].bits,
 348					      POWER_DOMAIN_NUM));
 349}
 350
 351static bool
 352__async_put_domains_state_ok(struct i915_power_domains *power_domains)
 353{
 354	struct drm_i915_private *i915 = container_of(power_domains,
 355						     struct drm_i915_private,
 356						     display.power.domains);
 357	struct intel_power_domain_mask async_put_mask;
 358	enum intel_display_power_domain domain;
 359	bool err = false;
 360
 361	err |= !assert_async_put_domain_masks_disjoint(power_domains);
 362	__async_put_domains_mask(power_domains, &async_put_mask);
 363	err |= drm_WARN_ON(&i915->drm,
 364			   !!power_domains->async_put_wakeref !=
 365			   !bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
 366
 367	for_each_power_domain(domain, &async_put_mask)
 368		err |= drm_WARN_ON(&i915->drm,
 369				   power_domains->domain_use_count[domain] != 1);
 370
 371	return !err;
 372}
 373
 374static void print_power_domains(struct i915_power_domains *power_domains,
 375				const char *prefix, struct intel_power_domain_mask *mask)
 376{
 377	struct drm_i915_private *i915 = container_of(power_domains,
 378						     struct drm_i915_private,
 379						     display.power.domains);
 380	enum intel_display_power_domain domain;
 381
 382	drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
 383	for_each_power_domain(domain, mask)
 384		drm_dbg(&i915->drm, "%s use_count %d\n",
 385			intel_display_power_domain_str(domain),
 386			power_domains->domain_use_count[domain]);
 387}
 388
 389static void
 390print_async_put_domains_state(struct i915_power_domains *power_domains)
 391{
 392	struct drm_i915_private *i915 = container_of(power_domains,
 393						     struct drm_i915_private,
 394						     display.power.domains);
 395
 396	drm_dbg(&i915->drm, "async_put_wakeref: %s\n",
 397		str_yes_no(power_domains->async_put_wakeref));
 398
 399	print_power_domains(power_domains, "async_put_domains[0]",
 400			    &power_domains->async_put_domains[0]);
 401	print_power_domains(power_domains, "async_put_domains[1]",
 402			    &power_domains->async_put_domains[1]);
 403}
 404
 405static void
 406verify_async_put_domains_state(struct i915_power_domains *power_domains)
 407{
 408	if (!__async_put_domains_state_ok(power_domains))
 409		print_async_put_domains_state(power_domains);
 410}
 411
 412#else
 413
 414static void
 415assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
 416{
 417}
 418
 419static void
 420verify_async_put_domains_state(struct i915_power_domains *power_domains)
 421{
 422}
 423
 424#endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
 425
 426static void async_put_domains_mask(struct i915_power_domains *power_domains,
 427				   struct intel_power_domain_mask *mask)
 428
 429{
 430	assert_async_put_domain_masks_disjoint(power_domains);
 431
 432	__async_put_domains_mask(power_domains, mask);
 433}
 434
 435static void
 436async_put_domains_clear_domain(struct i915_power_domains *power_domains,
 437			       enum intel_display_power_domain domain)
 438{
 439	assert_async_put_domain_masks_disjoint(power_domains);
 440
 441	clear_bit(domain, power_domains->async_put_domains[0].bits);
 442	clear_bit(domain, power_domains->async_put_domains[1].bits);
 443}
 444
 445static void
 446cancel_async_put_work(struct i915_power_domains *power_domains, bool sync)
 447{
 448	if (sync)
 449		cancel_delayed_work_sync(&power_domains->async_put_work);
 450	else
 451		cancel_delayed_work(&power_domains->async_put_work);
 452
 453	power_domains->async_put_next_delay = 0;
 454}
 455
 456static bool
 457intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
 458				       enum intel_display_power_domain domain)
 459{
 460	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 461	struct intel_power_domain_mask async_put_mask;
 462	bool ret = false;
 463
 464	async_put_domains_mask(power_domains, &async_put_mask);
 465	if (!test_bit(domain, async_put_mask.bits))
 466		goto out_verify;
 467
 468	async_put_domains_clear_domain(power_domains, domain);
 469
 470	ret = true;
 471
 472	async_put_domains_mask(power_domains, &async_put_mask);
 473	if (!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM))
 474		goto out_verify;
 475
 476	cancel_async_put_work(power_domains, false);
 477	intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
 478				 fetch_and_zero(&power_domains->async_put_wakeref));
 479out_verify:
 480	verify_async_put_domains_state(power_domains);
 481
 482	return ret;
 483}
 484
 485static void
 486__intel_display_power_get_domain(struct drm_i915_private *dev_priv,
 487				 enum intel_display_power_domain domain)
 488{
 489	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 490	struct i915_power_well *power_well;
 491
 492	if (intel_display_power_grab_async_put_ref(dev_priv, domain))
 493		return;
 494
 495	for_each_power_domain_well(dev_priv, power_well, domain)
 496		intel_power_well_get(dev_priv, power_well);
 497
 498	power_domains->domain_use_count[domain]++;
 499}
 500
 501/**
 502 * intel_display_power_get - grab a power domain reference
 503 * @dev_priv: i915 device instance
 504 * @domain: power domain to reference
 505 *
 506 * This function grabs a power domain reference for @domain and ensures that the
 507 * power domain and all its parents are powered up. Therefore users should only
 508 * grab a reference to the innermost power domain they need.
 509 *
 510 * Any power domain reference obtained by this function must have a symmetric
 511 * call to intel_display_power_put() to release the reference again.
 512 */
 513intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
 514					enum intel_display_power_domain domain)
 515{
 516	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 517	intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
 518
 519	mutex_lock(&power_domains->lock);
 520	__intel_display_power_get_domain(dev_priv, domain);
 521	mutex_unlock(&power_domains->lock);
 522
 523	return wakeref;
 524}
 525
 526/**
 527 * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
 528 * @dev_priv: i915 device instance
 529 * @domain: power domain to reference
 530 *
 531 * This function grabs a power domain reference for @domain and ensures that the
 532 * power domain and all its parents are powered up. Therefore users should only
 533 * grab a reference to the innermost power domain they need.
 534 *
 535 * Any power domain reference obtained by this function must have a symmetric
 536 * call to intel_display_power_put() to release the reference again.
 537 */
 538intel_wakeref_t
 539intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
 540				   enum intel_display_power_domain domain)
 541{
 542	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 543	intel_wakeref_t wakeref;
 544	bool is_enabled;
 545
 546	wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
 547	if (!wakeref)
 548		return NULL;
 549
 550	mutex_lock(&power_domains->lock);
 551
 552	if (__intel_display_power_is_enabled(dev_priv, domain)) {
 553		__intel_display_power_get_domain(dev_priv, domain);
 554		is_enabled = true;
 555	} else {
 556		is_enabled = false;
 557	}
 558
 559	mutex_unlock(&power_domains->lock);
 560
 561	if (!is_enabled) {
 562		intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
 563		wakeref = NULL;
 564	}
 565
 566	return wakeref;
 567}
 568
 569static void
 570__intel_display_power_put_domain(struct drm_i915_private *dev_priv,
 571				 enum intel_display_power_domain domain)
 572{
 573	struct i915_power_domains *power_domains;
 574	struct i915_power_well *power_well;
 575	const char *name = intel_display_power_domain_str(domain);
 576	struct intel_power_domain_mask async_put_mask;
 577
 578	power_domains = &dev_priv->display.power.domains;
 579
 580	drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
 581		 "Use count on domain %s is already zero\n",
 582		 name);
 583	async_put_domains_mask(power_domains, &async_put_mask);
 584	drm_WARN(&dev_priv->drm,
 585		 test_bit(domain, async_put_mask.bits),
 586		 "Async disabling of domain %s is pending\n",
 587		 name);
 588
 589	power_domains->domain_use_count[domain]--;
 590
 591	for_each_power_domain_well_reverse(dev_priv, power_well, domain)
 592		intel_power_well_put(dev_priv, power_well);
 593}
 594
 595static void __intel_display_power_put(struct drm_i915_private *dev_priv,
 596				      enum intel_display_power_domain domain)
 597{
 598	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 599
 600	mutex_lock(&power_domains->lock);
 601	__intel_display_power_put_domain(dev_priv, domain);
 602	mutex_unlock(&power_domains->lock);
 603}
 604
 605static void
 606queue_async_put_domains_work(struct i915_power_domains *power_domains,
 607			     intel_wakeref_t wakeref,
 608			     int delay_ms)
 609{
 610	struct drm_i915_private *i915 = container_of(power_domains,
 611						     struct drm_i915_private,
 612						     display.power.domains);
 613	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
 614	power_domains->async_put_wakeref = wakeref;
 615	drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
 616						    &power_domains->async_put_work,
 617						    msecs_to_jiffies(delay_ms)));
 618}
 619
 620static void
 621release_async_put_domains(struct i915_power_domains *power_domains,
 622			  struct intel_power_domain_mask *mask)
 623{
 624	struct drm_i915_private *dev_priv =
 625		container_of(power_domains, struct drm_i915_private,
 626			     display.power.domains);
 627	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
 628	enum intel_display_power_domain domain;
 629	intel_wakeref_t wakeref;
 630
 631	wakeref = intel_runtime_pm_get_noresume(rpm);
 632
 633	for_each_power_domain(domain, mask) {
 634		/* Clear before put, so put's sanity check is happy. */
 635		async_put_domains_clear_domain(power_domains, domain);
 636		__intel_display_power_put_domain(dev_priv, domain);
 637	}
 638
 639	intel_runtime_pm_put(rpm, wakeref);
 640}
 641
 642static void
 643intel_display_power_put_async_work(struct work_struct *work)
 644{
 645	struct drm_i915_private *dev_priv =
 646		container_of(work, struct drm_i915_private,
 647			     display.power.domains.async_put_work.work);
 648	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 649	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
 650	intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
 651	intel_wakeref_t old_work_wakeref = NULL;
 652
 653	mutex_lock(&power_domains->lock);
 654
 655	/*
 656	 * Bail out if all the domain refs pending to be released were grabbed
 657	 * by subsequent gets or a flush_work.
 658	 */
 659	old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
 660	if (!old_work_wakeref)
 661		goto out_verify;
 662
 663	release_async_put_domains(power_domains,
 664				  &power_domains->async_put_domains[0]);
 665
 666	/*
 667	 * Cancel the work that got queued after this one got dequeued,
 668	 * since here we released the corresponding async-put reference.
 669	 */
 670	cancel_async_put_work(power_domains, false);
 671
 672	/* Requeue the work if more domains were async put meanwhile. */
 673	if (!bitmap_empty(power_domains->async_put_domains[1].bits, POWER_DOMAIN_NUM)) {
 674		bitmap_copy(power_domains->async_put_domains[0].bits,
 675			    power_domains->async_put_domains[1].bits,
 676			    POWER_DOMAIN_NUM);
 677		bitmap_zero(power_domains->async_put_domains[1].bits,
 678			    POWER_DOMAIN_NUM);
 679		queue_async_put_domains_work(power_domains,
 680					     fetch_and_zero(&new_work_wakeref),
 681					     power_domains->async_put_next_delay);
 682		power_domains->async_put_next_delay = 0;
 683	}
 684
 685out_verify:
 686	verify_async_put_domains_state(power_domains);
 687
 688	mutex_unlock(&power_domains->lock);
 689
 690	if (old_work_wakeref)
 691		intel_runtime_pm_put_raw(rpm, old_work_wakeref);
 692	if (new_work_wakeref)
 693		intel_runtime_pm_put_raw(rpm, new_work_wakeref);
 694}
 695
 696/**
 697 * __intel_display_power_put_async - release a power domain reference asynchronously
 698 * @i915: i915 device instance
 699 * @domain: power domain to reference
 700 * @wakeref: wakeref acquired for the reference that is being released
 701 * @delay_ms: delay of powering down the power domain
 702 *
 703 * This function drops the power domain reference obtained by
 704 * intel_display_power_get*() and schedules a work to power down the
 705 * corresponding hardware block if this is the last reference.
 706 * The power down is delayed by @delay_ms if this is >= 0, or by a default
 707 * 100 ms otherwise.
 708 */
 709void __intel_display_power_put_async(struct drm_i915_private *i915,
 710				     enum intel_display_power_domain domain,
 711				     intel_wakeref_t wakeref,
 712				     int delay_ms)
 713{
 714	struct i915_power_domains *power_domains = &i915->display.power.domains;
 715	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 716	intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
 717
 718	delay_ms = delay_ms >= 0 ? delay_ms : 100;
 719
 720	mutex_lock(&power_domains->lock);
 721
 722	if (power_domains->domain_use_count[domain] > 1) {
 723		__intel_display_power_put_domain(i915, domain);
 724
 725		goto out_verify;
 726	}
 727
 728	drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
 729
 730	/* Let a pending work requeue itself or queue a new one. */
 731	if (power_domains->async_put_wakeref) {
 732		set_bit(domain, power_domains->async_put_domains[1].bits);
 733		power_domains->async_put_next_delay = max(power_domains->async_put_next_delay,
 734							  delay_ms);
 735	} else {
 736		set_bit(domain, power_domains->async_put_domains[0].bits);
 737		queue_async_put_domains_work(power_domains,
 738					     fetch_and_zero(&work_wakeref),
 739					     delay_ms);
 740	}
 741
 742out_verify:
 743	verify_async_put_domains_state(power_domains);
 744
 745	mutex_unlock(&power_domains->lock);
 746
 747	if (work_wakeref)
 748		intel_runtime_pm_put_raw(rpm, work_wakeref);
 749
 750	intel_runtime_pm_put(rpm, wakeref);
 751}
 752
 753/**
 754 * intel_display_power_flush_work - flushes the async display power disabling work
 755 * @i915: i915 device instance
 756 *
 757 * Flushes any pending work that was scheduled by a preceding
 758 * intel_display_power_put_async() call, completing the disabling of the
 759 * corresponding power domains.
 760 *
 761 * Note that the work handler function may still be running after this
 762 * function returns; to ensure that the work handler isn't running use
 763 * intel_display_power_flush_work_sync() instead.
 764 */
 765void intel_display_power_flush_work(struct drm_i915_private *i915)
 766{
 767	struct i915_power_domains *power_domains = &i915->display.power.domains;
 768	struct intel_power_domain_mask async_put_mask;
 769	intel_wakeref_t work_wakeref;
 770
 771	mutex_lock(&power_domains->lock);
 772
 773	work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
 774	if (!work_wakeref)
 775		goto out_verify;
 776
 777	async_put_domains_mask(power_domains, &async_put_mask);
 778	release_async_put_domains(power_domains, &async_put_mask);
 779	cancel_async_put_work(power_domains, false);
 780
 781out_verify:
 782	verify_async_put_domains_state(power_domains);
 783
 784	mutex_unlock(&power_domains->lock);
 785
 786	if (work_wakeref)
 787		intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
 788}
 789
 790/**
 791 * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
 792 * @i915: i915 device instance
 793 *
 794 * Like intel_display_power_flush_work(), but also ensure that the work
 795 * handler function is not running any more when this function returns.
 796 */
 797static void
 798intel_display_power_flush_work_sync(struct drm_i915_private *i915)
 799{
 800	struct i915_power_domains *power_domains = &i915->display.power.domains;
 801
 802	intel_display_power_flush_work(i915);
 803	cancel_async_put_work(power_domains, true);
 804
 805	verify_async_put_domains_state(power_domains);
 806
 807	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
 808}
 809
 810#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 811/**
 812 * intel_display_power_put - release a power domain reference
 813 * @dev_priv: i915 device instance
 814 * @domain: power domain to reference
 815 * @wakeref: wakeref acquired for the reference that is being released
 816 *
 817 * This function drops the power domain reference obtained by
 818 * intel_display_power_get() and might power down the corresponding hardware
 819 * block right away if this is the last reference.
 820 */
 821void intel_display_power_put(struct drm_i915_private *dev_priv,
 822			     enum intel_display_power_domain domain,
 823			     intel_wakeref_t wakeref)
 824{
 825	__intel_display_power_put(dev_priv, domain);
 826	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
 827}
 828#else
 829/**
 830 * intel_display_power_put_unchecked - release an unchecked power domain reference
 831 * @dev_priv: i915 device instance
 832 * @domain: power domain to reference
 833 *
 834 * This function drops the power domain reference obtained by
 835 * intel_display_power_get() and might power down the corresponding hardware
 836 * block right away if this is the last reference.
 837 *
 838 * This function is only for the power domain code's internal use to suppress wakeref
 839 * tracking when the correspondig debug kconfig option is disabled, should not
 840 * be used otherwise.
 841 */
 842void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
 843				       enum intel_display_power_domain domain)
 844{
 845	__intel_display_power_put(dev_priv, domain);
 846	intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
 847}
 848#endif
 849
 850void
 851intel_display_power_get_in_set(struct drm_i915_private *i915,
 852			       struct intel_display_power_domain_set *power_domain_set,
 853			       enum intel_display_power_domain domain)
 854{
 855	intel_wakeref_t __maybe_unused wf;
 856
 857	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
 858
 859	wf = intel_display_power_get(i915, domain);
 860#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 861	power_domain_set->wakerefs[domain] = wf;
 862#endif
 863	set_bit(domain, power_domain_set->mask.bits);
 864}
 865
 866bool
 867intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
 868					  struct intel_display_power_domain_set *power_domain_set,
 869					  enum intel_display_power_domain domain)
 870{
 871	intel_wakeref_t wf;
 872
 873	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
 874
 875	wf = intel_display_power_get_if_enabled(i915, domain);
 876	if (!wf)
 877		return false;
 878
 879#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 880	power_domain_set->wakerefs[domain] = wf;
 881#endif
 882	set_bit(domain, power_domain_set->mask.bits);
 883
 884	return true;
 885}
 886
 887void
 888intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
 889				    struct intel_display_power_domain_set *power_domain_set,
 890				    struct intel_power_domain_mask *mask)
 891{
 892	enum intel_display_power_domain domain;
 893
 894	drm_WARN_ON(&i915->drm,
 895		    !bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
 896
 897	for_each_power_domain(domain, mask) {
 898		intel_wakeref_t __maybe_unused wf = INTEL_WAKEREF_DEF;
 899
 900#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 901		wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
 902#endif
 903		intel_display_power_put(i915, domain, wf);
 904		clear_bit(domain, power_domain_set->mask.bits);
 905	}
 906}
 907
 908static int
 909sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
 910				   int disable_power_well)
 911{
 912	if (disable_power_well >= 0)
 913		return !!disable_power_well;
 914
 915	return 1;
 916}
 917
 918static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 919			       int enable_dc)
 920{
 921	u32 mask;
 922	int requested_dc;
 923	int max_dc;
 924
 925	if (!HAS_DISPLAY(dev_priv))
 926		return 0;
 927
 928	if (DISPLAY_VER(dev_priv) >= 20)
 929		max_dc = 2;
 930	else if (IS_DG2(dev_priv))
 931		max_dc = 1;
 932	else if (IS_DG1(dev_priv))
 933		max_dc = 3;
 934	else if (DISPLAY_VER(dev_priv) >= 12)
 935		max_dc = 4;
 936	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 937		max_dc = 1;
 938	else if (DISPLAY_VER(dev_priv) >= 9)
 939		max_dc = 2;
 940	else
 941		max_dc = 0;
 942
 943	/*
 944	 * DC9 has a separate HW flow from the rest of the DC states,
 945	 * not depending on the DMC firmware. It's needed by system
 946	 * suspend/resume, so allow it unconditionally.
 947	 */
 948	mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
 949		DISPLAY_VER(dev_priv) >= 11 ?
 950	       DC_STATE_EN_DC9 : 0;
 951
 952	if (!dev_priv->display.params.disable_power_well)
 953		max_dc = 0;
 954
 955	if (enable_dc >= 0 && enable_dc <= max_dc) {
 956		requested_dc = enable_dc;
 957	} else if (enable_dc == -1) {
 958		requested_dc = max_dc;
 959	} else if (enable_dc > max_dc && enable_dc <= 4) {
 960		drm_dbg_kms(&dev_priv->drm,
 961			    "Adjusting requested max DC state (%d->%d)\n",
 962			    enable_dc, max_dc);
 963		requested_dc = max_dc;
 964	} else {
 965		drm_err(&dev_priv->drm,
 966			"Unexpected value for enable_dc (%d)\n", enable_dc);
 967		requested_dc = max_dc;
 968	}
 969
 970	switch (requested_dc) {
 971	case 4:
 972		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
 973		break;
 974	case 3:
 975		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
 976		break;
 977	case 2:
 978		mask |= DC_STATE_EN_UPTO_DC6;
 979		break;
 980	case 1:
 981		mask |= DC_STATE_EN_UPTO_DC5;
 982		break;
 983	}
 984
 985	drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
 986
 987	return mask;
 988}
 989
 990/**
 991 * intel_power_domains_init - initializes the power domain structures
 992 * @dev_priv: i915 device instance
 993 *
 994 * Initializes the power domain structures for @dev_priv depending upon the
 995 * supported platform.
 996 */
 997int intel_power_domains_init(struct drm_i915_private *dev_priv)
 998{
 999	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1000
1001	dev_priv->display.params.disable_power_well =
1002		sanitize_disable_power_well_option(dev_priv,
1003						   dev_priv->display.params.disable_power_well);
1004	power_domains->allowed_dc_mask =
1005		get_allowed_dc_mask(dev_priv, dev_priv->display.params.enable_dc);
1006
1007	power_domains->target_dc_state =
1008		sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1009
1010	mutex_init(&power_domains->lock);
1011
1012	INIT_DELAYED_WORK(&power_domains->async_put_work,
1013			  intel_display_power_put_async_work);
1014
1015	return intel_display_power_map_init(power_domains);
1016}
1017
1018/**
1019 * intel_power_domains_cleanup - clean up power domains resources
1020 * @dev_priv: i915 device instance
1021 *
1022 * Release any resources acquired by intel_power_domains_init()
1023 */
1024void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
1025{
1026	intel_display_power_map_cleanup(&dev_priv->display.power.domains);
1027}
1028
1029static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
1030{
1031	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1032	struct i915_power_well *power_well;
1033
1034	mutex_lock(&power_domains->lock);
1035	for_each_power_well(dev_priv, power_well)
1036		intel_power_well_sync_hw(dev_priv, power_well);
1037	mutex_unlock(&power_domains->lock);
1038}
1039
1040static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
1041				enum dbuf_slice slice, bool enable)
1042{
1043	i915_reg_t reg = DBUF_CTL_S(slice);
1044	bool state;
1045
1046	intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
1047		     enable ? DBUF_POWER_REQUEST : 0);
1048	intel_de_posting_read(dev_priv, reg);
1049	udelay(10);
1050
1051	state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
1052	drm_WARN(&dev_priv->drm, enable != state,
1053		 "DBuf slice %d power %s timeout!\n",
1054		 slice, str_enable_disable(enable));
1055}
1056
1057void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
1058			     u8 req_slices)
1059{
1060	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1061	u8 slice_mask = DISPLAY_INFO(dev_priv)->dbuf.slice_mask;
1062	enum dbuf_slice slice;
1063
1064	drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
1065		 "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1066		 req_slices, slice_mask);
1067
1068	drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
1069		    req_slices);
1070
1071	/*
1072	 * Might be running this in parallel to gen9_dc_off_power_well_enable
1073	 * being called from intel_dp_detect for instance,
1074	 * which causes assertion triggered by race condition,
1075	 * as gen9_assert_dbuf_enabled might preempt this when registers
1076	 * were already updated, while dev_priv was not.
1077	 */
1078	mutex_lock(&power_domains->lock);
1079
1080	for_each_dbuf_slice(dev_priv, slice)
1081		gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
1082
1083	dev_priv->display.dbuf.enabled_slices = req_slices;
1084
1085	mutex_unlock(&power_domains->lock);
1086}
1087
1088static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
1089{
1090	u8 slices_mask;
1091
1092	dev_priv->display.dbuf.enabled_slices =
1093		intel_enabled_dbuf_slices_mask(dev_priv);
1094
1095	slices_mask = BIT(DBUF_S1) | dev_priv->display.dbuf.enabled_slices;
1096
1097	if (DISPLAY_VER(dev_priv) >= 14)
1098		intel_pmdemand_program_dbuf(dev_priv, slices_mask);
1099
1100	/*
1101	 * Just power up at least 1 slice, we will
1102	 * figure out later which slices we have and what we need.
1103	 */
1104	gen9_dbuf_slices_update(dev_priv, slices_mask);
1105}
1106
1107static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
1108{
1109	gen9_dbuf_slices_update(dev_priv, 0);
1110
1111	if (DISPLAY_VER(dev_priv) >= 14)
1112		intel_pmdemand_program_dbuf(dev_priv, 0);
1113}
1114
1115static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
1116{
1117	enum dbuf_slice slice;
1118
1119	if (IS_ALDERLAKE_P(dev_priv))
1120		return;
1121
1122	for_each_dbuf_slice(dev_priv, slice)
1123		intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
1124			     DBUF_TRACKER_STATE_SERVICE_MASK,
1125			     DBUF_TRACKER_STATE_SERVICE(8));
1126}
1127
1128static void icl_mbus_init(struct drm_i915_private *dev_priv)
1129{
1130	unsigned long abox_regs = DISPLAY_INFO(dev_priv)->abox_mask;
1131	u32 mask, val, i;
1132
1133	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1134		return;
1135
1136	mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1137		MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1138		MBUS_ABOX_B_CREDIT_MASK |
1139		MBUS_ABOX_BW_CREDIT_MASK;
1140	val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1141		MBUS_ABOX_BT_CREDIT_POOL2(16) |
1142		MBUS_ABOX_B_CREDIT(1) |
1143		MBUS_ABOX_BW_CREDIT(1);
1144
1145	/*
1146	 * gen12 platforms that use abox1 and abox2 for pixel data reads still
1147	 * expect us to program the abox_ctl0 register as well, even though
1148	 * we don't have to program other instance-0 registers like BW_BUDDY.
1149	 */
1150	if (DISPLAY_VER(dev_priv) == 12)
1151		abox_regs |= BIT(0);
1152
1153	for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
1154		intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
1155}
1156
1157static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
1158{
1159	u32 val = intel_de_read(dev_priv, LCPLL_CTL);
1160
1161	/*
1162	 * The LCPLL register should be turned on by the BIOS. For now
1163	 * let's just check its state and print errors in case
1164	 * something is wrong.  Don't even try to turn it on.
1165	 */
1166
1167	if (val & LCPLL_CD_SOURCE_FCLK)
1168		drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
1169
1170	if (val & LCPLL_PLL_DISABLE)
1171		drm_err(&dev_priv->drm, "LCPLL is disabled\n");
1172
1173	if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1174		drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
1175}
1176
1177static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
1178{
1179	struct intel_display *display = &dev_priv->display;
1180	struct intel_crtc *crtc;
1181
1182	for_each_intel_crtc(display->drm, crtc)
1183		INTEL_DISPLAY_STATE_WARN(display, crtc->active,
1184					 "CRTC for pipe %c enabled\n",
1185					 pipe_name(crtc->pipe));
1186
1187	INTEL_DISPLAY_STATE_WARN(display, intel_de_read(display, HSW_PWR_WELL_CTL2),
1188				 "Display power well on\n");
1189	INTEL_DISPLAY_STATE_WARN(display,
1190				 intel_de_read(display, SPLL_CTL) & SPLL_PLL_ENABLE,
1191				 "SPLL enabled\n");
1192	INTEL_DISPLAY_STATE_WARN(display,
1193				 intel_de_read(display, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1194				 "WRPLL1 enabled\n");
1195	INTEL_DISPLAY_STATE_WARN(display,
1196				 intel_de_read(display, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1197				 "WRPLL2 enabled\n");
1198	INTEL_DISPLAY_STATE_WARN(display,
1199				 intel_de_read(display, PP_STATUS(display, 0)) & PP_ON,
1200				 "Panel power on\n");
1201	INTEL_DISPLAY_STATE_WARN(display,
1202				 intel_de_read(display, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1203				 "CPU PWM1 enabled\n");
1204	if (IS_HASWELL(dev_priv))
1205		INTEL_DISPLAY_STATE_WARN(display,
1206					 intel_de_read(display, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1207					 "CPU PWM2 enabled\n");
1208	INTEL_DISPLAY_STATE_WARN(display,
1209				 intel_de_read(display, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1210				 "PCH PWM1 enabled\n");
1211	INTEL_DISPLAY_STATE_WARN(display,
1212				 (intel_de_read(display, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1213				 "Utility pin enabled in PWM mode\n");
1214	INTEL_DISPLAY_STATE_WARN(display,
1215				 intel_de_read(display, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1216				 "PCH GTC enabled\n");
1217
1218	/*
1219	 * In theory we can still leave IRQs enabled, as long as only the HPD
1220	 * interrupts remain enabled. We used to check for that, but since it's
1221	 * gen-specific and since we only disable LCPLL after we fully disable
1222	 * the interrupts, the check below should be enough.
1223	 */
1224	INTEL_DISPLAY_STATE_WARN(display, intel_irqs_enabled(dev_priv),
1225				 "IRQs enabled\n");
1226}
1227
1228static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
1229{
1230	if (IS_HASWELL(dev_priv))
1231		return intel_de_read(dev_priv, D_COMP_HSW);
1232	else
1233		return intel_de_read(dev_priv, D_COMP_BDW);
1234}
1235
1236static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
1237{
1238	if (IS_HASWELL(dev_priv)) {
1239		if (snb_pcode_write(&dev_priv->uncore, GEN6_PCODE_WRITE_D_COMP, val))
1240			drm_dbg_kms(&dev_priv->drm,
1241				    "Failed to write to D_COMP\n");
1242	} else {
1243		intel_de_write(dev_priv, D_COMP_BDW, val);
1244		intel_de_posting_read(dev_priv, D_COMP_BDW);
1245	}
1246}
1247
1248/*
1249 * This function implements pieces of two sequences from BSpec:
1250 * - Sequence for display software to disable LCPLL
1251 * - Sequence for display software to allow package C8+
1252 * The steps implemented here are just the steps that actually touch the LCPLL
1253 * register. Callers should take care of disabling all the display engine
1254 * functions, doing the mode unset, fixing interrupts, etc.
1255 */
1256static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
1257			      bool switch_to_fclk, bool allow_power_down)
1258{
1259	u32 val;
1260
1261	assert_can_disable_lcpll(dev_priv);
1262
1263	val = intel_de_read(dev_priv, LCPLL_CTL);
1264
1265	if (switch_to_fclk) {
1266		val |= LCPLL_CD_SOURCE_FCLK;
1267		intel_de_write(dev_priv, LCPLL_CTL, val);
1268
1269		if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
1270				LCPLL_CD_SOURCE_FCLK_DONE, 1))
1271			drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
1272
1273		val = intel_de_read(dev_priv, LCPLL_CTL);
1274	}
1275
1276	val |= LCPLL_PLL_DISABLE;
1277	intel_de_write(dev_priv, LCPLL_CTL, val);
1278	intel_de_posting_read(dev_priv, LCPLL_CTL);
1279
1280	if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
1281		drm_err(&dev_priv->drm, "LCPLL still locked\n");
1282
1283	val = hsw_read_dcomp(dev_priv);
1284	val |= D_COMP_COMP_DISABLE;
1285	hsw_write_dcomp(dev_priv, val);
1286	ndelay(100);
1287
1288	if (wait_for((hsw_read_dcomp(dev_priv) &
1289		      D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
1290		drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
1291
1292	if (allow_power_down) {
1293		intel_de_rmw(dev_priv, LCPLL_CTL, 0, LCPLL_POWER_DOWN_ALLOW);
1294		intel_de_posting_read(dev_priv, LCPLL_CTL);
1295	}
1296}
1297
1298/*
1299 * Fully restores LCPLL, disallowing power down and switching back to LCPLL
1300 * source.
1301 */
1302static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
1303{
1304	struct intel_display *display = &dev_priv->display;
1305	u32 val;
1306
1307	val = intel_de_read(dev_priv, LCPLL_CTL);
1308
1309	if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1310		    LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1311		return;
1312
1313	/*
1314	 * Make sure we're not on PC8 state before disabling PC8, otherwise
1315	 * we'll hang the machine. To prevent PC8 state, just enable force_wake.
1316	 */
1317	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1318
1319	if (val & LCPLL_POWER_DOWN_ALLOW) {
1320		val &= ~LCPLL_POWER_DOWN_ALLOW;
1321		intel_de_write(dev_priv, LCPLL_CTL, val);
1322		intel_de_posting_read(dev_priv, LCPLL_CTL);
1323	}
1324
1325	val = hsw_read_dcomp(dev_priv);
1326	val |= D_COMP_COMP_FORCE;
1327	val &= ~D_COMP_COMP_DISABLE;
1328	hsw_write_dcomp(dev_priv, val);
1329
1330	val = intel_de_read(dev_priv, LCPLL_CTL);
1331	val &= ~LCPLL_PLL_DISABLE;
1332	intel_de_write(dev_priv, LCPLL_CTL, val);
1333
1334	if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
1335		drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
1336
1337	if (val & LCPLL_CD_SOURCE_FCLK) {
1338		intel_de_rmw(dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, 0);
1339
1340		if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
1341				 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
1342			drm_err(&dev_priv->drm,
1343				"Switching back to LCPLL failed\n");
1344	}
1345
1346	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1347
1348	intel_update_cdclk(display);
1349	intel_cdclk_dump_config(display, &display->cdclk.hw, "Current CDCLK");
1350}
1351
1352/*
1353 * Package states C8 and deeper are really deep PC states that can only be
1354 * reached when all the devices on the system allow it, so even if the graphics
1355 * device allows PC8+, it doesn't mean the system will actually get to these
1356 * states. Our driver only allows PC8+ when going into runtime PM.
1357 *
1358 * The requirements for PC8+ are that all the outputs are disabled, the power
1359 * well is disabled and most interrupts are disabled, and these are also
1360 * requirements for runtime PM. When these conditions are met, we manually do
1361 * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1362 * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1363 * hang the machine.
1364 *
1365 * When we really reach PC8 or deeper states (not just when we allow it) we lose
1366 * the state of some registers, so when we come back from PC8+ we need to
1367 * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1368 * need to take care of the registers kept by RC6. Notice that this happens even
1369 * if we don't put the device in PCI D3 state (which is what currently happens
1370 * because of the runtime PM support).
1371 *
1372 * For more, read "Display Sequences for Package C8" on the hardware
1373 * documentation.
1374 */
1375static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
1376{
1377	drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
1378
1379	if (HAS_PCH_LPT_LP(dev_priv))
1380		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1381			     PCH_LP_PARTITION_LEVEL_DISABLE, 0);
1382
1383	lpt_disable_clkout_dp(dev_priv);
1384	hsw_disable_lcpll(dev_priv, true, true);
1385}
1386
1387static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
1388{
1389	drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
1390
1391	hsw_restore_lcpll(dev_priv);
1392	intel_init_pch_refclk(dev_priv);
1393
1394	/* Many display registers don't survive PC8+ */
1395	intel_clock_gating_init(dev_priv);
1396}
1397
1398static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
1399				      bool enable)
1400{
1401	i915_reg_t reg;
1402	u32 reset_bits;
1403
1404	if (IS_IVYBRIDGE(dev_priv)) {
1405		reg = GEN7_MSG_CTL;
1406		reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1407	} else {
1408		reg = HSW_NDE_RSTWRN_OPT;
1409		reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1410	}
1411
1412	if (DISPLAY_VER(dev_priv) >= 14)
1413		reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1414
1415	intel_de_rmw(dev_priv, reg, reset_bits, enable ? reset_bits : 0);
1416}
1417
1418static void skl_display_core_init(struct drm_i915_private *dev_priv,
1419				  bool resume)
1420{
1421	struct intel_display *display = &dev_priv->display;
1422	struct i915_power_domains *power_domains = &display->power.domains;
1423	struct i915_power_well *well;
1424
1425	gen9_set_dc_state(display, DC_STATE_DISABLE);
1426
1427	/* enable PCH reset handshake */
1428	intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1429
1430	if (!HAS_DISPLAY(dev_priv))
1431		return;
1432
1433	/* enable PG1 and Misc I/O */
1434	mutex_lock(&power_domains->lock);
1435
1436	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1437	intel_power_well_enable(dev_priv, well);
1438
1439	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
1440	intel_power_well_enable(dev_priv, well);
1441
1442	mutex_unlock(&power_domains->lock);
1443
1444	intel_cdclk_init_hw(display);
1445
1446	gen9_dbuf_enable(dev_priv);
1447
1448	if (resume)
1449		intel_dmc_load_program(display);
1450}
1451
1452static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
1453{
1454	struct intel_display *display = &dev_priv->display;
1455	struct i915_power_domains *power_domains = &display->power.domains;
1456	struct i915_power_well *well;
1457
1458	if (!HAS_DISPLAY(dev_priv))
1459		return;
1460
1461	gen9_disable_dc_states(display);
1462	/* TODO: disable DMC program */
1463
1464	gen9_dbuf_disable(dev_priv);
1465
1466	intel_cdclk_uninit_hw(display);
1467
1468	/* The spec doesn't call for removing the reset handshake flag */
1469	/* disable PG1 and Misc I/O */
1470
1471	mutex_lock(&power_domains->lock);
1472
1473	/*
1474	 * BSpec says to keep the MISC IO power well enabled here, only
1475	 * remove our request for power well 1.
1476	 * Note that even though the driver's request is removed power well 1
1477	 * may stay enabled after this due to DMC's own request on it.
1478	 */
1479	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1480	intel_power_well_disable(dev_priv, well);
1481
1482	mutex_unlock(&power_domains->lock);
1483
1484	usleep_range(10, 30);		/* 10 us delay per Bspec */
1485}
1486
1487static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
1488{
1489	struct intel_display *display = &dev_priv->display;
1490	struct i915_power_domains *power_domains = &display->power.domains;
1491	struct i915_power_well *well;
1492
1493	gen9_set_dc_state(display, DC_STATE_DISABLE);
1494
1495	/*
1496	 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1497	 * or else the reset will hang because there is no PCH to respond.
1498	 * Move the handshake programming to initialization sequence.
1499	 * Previously was left up to BIOS.
1500	 */
1501	intel_pch_reset_handshake(dev_priv, false);
1502
1503	if (!HAS_DISPLAY(dev_priv))
1504		return;
1505
1506	/* Enable PG1 */
1507	mutex_lock(&power_domains->lock);
1508
1509	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1510	intel_power_well_enable(dev_priv, well);
1511
1512	mutex_unlock(&power_domains->lock);
1513
1514	intel_cdclk_init_hw(display);
1515
1516	gen9_dbuf_enable(dev_priv);
1517
1518	if (resume)
1519		intel_dmc_load_program(display);
1520}
1521
1522static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
1523{
1524	struct intel_display *display = &dev_priv->display;
1525	struct i915_power_domains *power_domains = &display->power.domains;
1526	struct i915_power_well *well;
1527
1528	if (!HAS_DISPLAY(dev_priv))
1529		return;
1530
1531	gen9_disable_dc_states(display);
1532	/* TODO: disable DMC program */
1533
1534	gen9_dbuf_disable(dev_priv);
1535
1536	intel_cdclk_uninit_hw(display);
1537
1538	/* The spec doesn't call for removing the reset handshake flag */
1539
1540	/*
1541	 * Disable PW1 (PG1).
1542	 * Note that even though the driver's request is removed power well 1
1543	 * may stay enabled after this due to DMC's own request on it.
1544	 */
1545	mutex_lock(&power_domains->lock);
1546
1547	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1548	intel_power_well_disable(dev_priv, well);
1549
1550	mutex_unlock(&power_domains->lock);
1551
1552	usleep_range(10, 30);		/* 10 us delay per Bspec */
1553}
1554
1555struct buddy_page_mask {
1556	u32 page_mask;
1557	u8 type;
1558	u8 num_channels;
1559};
1560
1561static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1562	{ .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0xF },
1563	{ .num_channels = 1, .type = INTEL_DRAM_DDR5,	.page_mask = 0xF },
1564	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1565	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1566	{ .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1F },
1567	{ .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1E },
1568	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1569	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1570	{}
1571};
1572
1573static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1574	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1575	{ .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1 },
1576	{ .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1 },
1577	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1578	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1579	{ .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x3 },
1580	{ .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x3 },
1581	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1582	{}
1583};
1584
1585static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
1586{
1587	enum intel_dram_type type = dev_priv->dram_info.type;
1588	u8 num_channels = dev_priv->dram_info.num_channels;
1589	const struct buddy_page_mask *table;
1590	unsigned long abox_mask = DISPLAY_INFO(dev_priv)->abox_mask;
1591	int config, i;
1592
1593	/* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1594	if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv))
1595		return;
1596
1597	if (IS_ALDERLAKE_S(dev_priv) ||
1598	    (IS_ROCKETLAKE(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)))
1599		/* Wa_1409767108 */
1600		table = wa_1409767108_buddy_page_masks;
1601	else
1602		table = tgl_buddy_page_masks;
1603
1604	for (config = 0; table[config].page_mask != 0; config++)
1605		if (table[config].num_channels == num_channels &&
1606		    table[config].type == type)
1607			break;
1608
1609	if (table[config].page_mask == 0) {
1610		drm_dbg(&dev_priv->drm,
1611			"Unknown memory configuration; disabling address buddy logic.\n");
1612		for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
1613			intel_de_write(dev_priv, BW_BUDDY_CTL(i),
1614				       BW_BUDDY_DISABLE);
1615	} else {
1616		for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
1617			intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
1618				       table[config].page_mask);
1619
1620			/* Wa_22010178259:tgl,dg1,rkl,adl-s */
1621			if (DISPLAY_VER(dev_priv) == 12)
1622				intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
1623					     BW_BUDDY_TLB_REQ_TIMER_MASK,
1624					     BW_BUDDY_TLB_REQ_TIMER(0x8));
1625		}
1626	}
1627}
1628
1629static void icl_display_core_init(struct drm_i915_private *dev_priv,
1630				  bool resume)
1631{
1632	struct intel_display *display = &dev_priv->display;
1633	struct i915_power_domains *power_domains = &display->power.domains;
1634	struct i915_power_well *well;
1635
1636	gen9_set_dc_state(display, DC_STATE_DISABLE);
1637
1638	/* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1639	if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
1640	    INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
1641		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
1642			     PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1643
1644	/* 1. Enable PCH reset handshake. */
1645	intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
1646
1647	if (!HAS_DISPLAY(dev_priv))
1648		return;
1649
1650	/* 2. Initialize all combo phys */
1651	intel_combo_phy_init(dev_priv);
1652
1653	/*
1654	 * 3. Enable Power Well 1 (PG1).
1655	 *    The AUX IO power wells will be enabled on demand.
1656	 */
1657	mutex_lock(&power_domains->lock);
1658	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1659	intel_power_well_enable(dev_priv, well);
1660	mutex_unlock(&power_domains->lock);
1661
1662	if (DISPLAY_VER(dev_priv) == 14)
1663		intel_de_rmw(dev_priv, DC_STATE_EN,
1664			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, 0);
1665
1666	/* 4. Enable CDCLK. */
1667	intel_cdclk_init_hw(display);
1668
1669	if (DISPLAY_VER(dev_priv) >= 12)
1670		gen12_dbuf_slices_config(dev_priv);
1671
1672	/* 5. Enable DBUF. */
1673	gen9_dbuf_enable(dev_priv);
1674
1675	/* 6. Setup MBUS. */
1676	icl_mbus_init(dev_priv);
1677
1678	/* 7. Program arbiter BW_BUDDY registers */
1679	if (DISPLAY_VER(dev_priv) >= 12)
1680		tgl_bw_buddy_init(dev_priv);
1681
1682	/* 8. Ensure PHYs have completed calibration and adaptation */
1683	if (IS_DG2(dev_priv))
1684		intel_snps_phy_wait_for_calibration(dev_priv);
1685
1686	/* 9. XE2_HPD: Program CHICKEN_MISC_2 before any cursor or planes are enabled */
1687	if (DISPLAY_VERx100(dev_priv) == 1401)
1688		intel_de_rmw(dev_priv, CHICKEN_MISC_2, BMG_DARB_HALF_BLK_END_BURST, 1);
1689
1690	if (resume)
1691		intel_dmc_load_program(display);
1692
1693	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p,dg2 */
1694	if (IS_DISPLAY_VERx100(dev_priv, 1200, 1300))
1695		intel_de_rmw(dev_priv, GEN11_CHICKEN_DCPR_2, 0,
1696			     DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1697			     DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1698
1699	/* Wa_14011503030:xelpd */
1700	if (DISPLAY_VER(dev_priv) == 13)
1701		intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
1702
1703	/* Wa_15013987218 */
1704	if (DISPLAY_VER(dev_priv) == 20) {
1705		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1706			     0, PCH_GMBUSUNIT_CLOCK_GATE_DISABLE);
1707		intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D,
1708			     PCH_GMBUSUNIT_CLOCK_GATE_DISABLE, 0);
1709	}
1710}
1711
1712static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
1713{
1714	struct intel_display *display = &dev_priv->display;
1715	struct i915_power_domains *power_domains = &display->power.domains;
1716	struct i915_power_well *well;
1717
1718	if (!HAS_DISPLAY(dev_priv))
1719		return;
1720
1721	gen9_disable_dc_states(display);
1722	intel_dmc_disable_program(display);
1723
1724	/* 1. Disable all display engine functions -> aready done */
1725
1726	/* 2. Disable DBUF */
1727	gen9_dbuf_disable(dev_priv);
1728
1729	/* 3. Disable CD clock */
1730	intel_cdclk_uninit_hw(display);
1731
1732	if (DISPLAY_VER(dev_priv) == 14)
1733		intel_de_rmw(dev_priv, DC_STATE_EN, 0,
1734			     HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1735
1736	/*
1737	 * 4. Disable Power Well 1 (PG1).
1738	 *    The AUX IO power wells are toggled on demand, so they are already
1739	 *    disabled at this point.
1740	 */
1741	mutex_lock(&power_domains->lock);
1742	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
1743	intel_power_well_disable(dev_priv, well);
1744	mutex_unlock(&power_domains->lock);
1745
1746	/* 5. */
1747	intel_combo_phy_uninit(dev_priv);
1748}
1749
1750static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1751{
1752	struct i915_power_well *cmn_bc =
1753		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1754	struct i915_power_well *cmn_d =
1755		lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1756
1757	/*
1758	 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
1759	 * workaround never ever read DISPLAY_PHY_CONTROL, and
1760	 * instead maintain a shadow copy ourselves. Use the actual
1761	 * power well state and lane status to reconstruct the
1762	 * expected initial value.
1763	 */
1764	dev_priv->display.power.chv_phy_control =
1765		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1766		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1767		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1768		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1769		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1770
1771	/*
1772	 * If all lanes are disabled we leave the override disabled
1773	 * with all power down bits cleared to match the state we
1774	 * would use after disabling the port. Otherwise enable the
1775	 * override and set the lane powerdown bits accding to the
1776	 * current lane status.
1777	 */
1778	if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1779		u32 status = intel_de_read(dev_priv, DPLL(dev_priv, PIPE_A));
1780		unsigned int mask;
1781
1782		mask = status & DPLL_PORTB_READY_MASK;
1783		if (mask == 0xf)
1784			mask = 0x0;
1785		else
1786			dev_priv->display.power.chv_phy_control |=
1787				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1788
1789		dev_priv->display.power.chv_phy_control |=
1790			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1791
1792		mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1793		if (mask == 0xf)
1794			mask = 0x0;
1795		else
1796			dev_priv->display.power.chv_phy_control |=
1797				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1798
1799		dev_priv->display.power.chv_phy_control |=
1800			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1801
1802		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1803
1804		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
1805	} else {
1806		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
1807	}
1808
1809	if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1810		u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
1811		unsigned int mask;
1812
1813		mask = status & DPLL_PORTD_READY_MASK;
1814
1815		if (mask == 0xf)
1816			mask = 0x0;
1817		else
1818			dev_priv->display.power.chv_phy_control |=
1819				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1820
1821		dev_priv->display.power.chv_phy_control |=
1822			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1823
1824		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1825
1826		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
1827	} else {
1828		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
1829	}
1830
1831	drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
1832		    dev_priv->display.power.chv_phy_control);
1833
1834	/* Defer application of initial phy_control to enabling the powerwell */
1835}
1836
1837static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1838{
1839	struct i915_power_well *cmn =
1840		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1841	struct i915_power_well *disp2d =
1842		lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
1843
1844	/* If the display might be already active skip this */
1845	if (intel_power_well_is_enabled(dev_priv, cmn) &&
1846	    intel_power_well_is_enabled(dev_priv, disp2d) &&
1847	    intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
1848		return;
1849
1850	drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
1851
1852	/* cmnlane needs DPLL registers */
1853	intel_power_well_enable(dev_priv, disp2d);
1854
1855	/*
1856	 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1857	 * Need to assert and de-assert PHY SB reset by gating the
1858	 * common lane power, then un-gating it.
1859	 * Simply ungating isn't enough to reset the PHY enough to get
1860	 * ports and lanes running.
1861	 */
1862	intel_power_well_disable(dev_priv, cmn);
1863}
1864
1865static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
1866{
1867	bool ret;
1868
1869	vlv_punit_get(dev_priv);
1870	ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1871	vlv_punit_put(dev_priv);
1872
1873	return ret;
1874}
1875
1876static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
1877{
1878	drm_WARN(&dev_priv->drm,
1879		 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
1880		 "VED not power gated\n");
1881}
1882
1883static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
1884{
1885	static const struct pci_device_id isp_ids[] = {
1886		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1887		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1888		{}
1889	};
1890
1891	drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
1892		 !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
1893		 "ISP not power gated\n");
1894}
1895
1896static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
1897
1898/**
1899 * intel_power_domains_init_hw - initialize hardware power domain state
1900 * @i915: i915 device instance
1901 * @resume: Called from resume code paths or not
1902 *
1903 * This function initializes the hardware power domain state and enables all
1904 * power wells belonging to the INIT power domain. Power wells in other
1905 * domains (and not in the INIT domain) are referenced or disabled by
1906 * intel_modeset_readout_hw_state(). After that the reference count of each
1907 * power well must match its HW enabled state, see
1908 * intel_power_domains_verify_state().
1909 *
1910 * It will return with power domains disabled (to be enabled later by
1911 * intel_power_domains_enable()) and must be paired with
1912 * intel_power_domains_driver_remove().
1913 */
1914void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
1915{
1916	struct i915_power_domains *power_domains = &i915->display.power.domains;
1917
1918	power_domains->initializing = true;
1919
1920	if (DISPLAY_VER(i915) >= 11) {
1921		icl_display_core_init(i915, resume);
1922	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
1923		bxt_display_core_init(i915, resume);
1924	} else if (DISPLAY_VER(i915) == 9) {
1925		skl_display_core_init(i915, resume);
1926	} else if (IS_CHERRYVIEW(i915)) {
1927		mutex_lock(&power_domains->lock);
1928		chv_phy_control_init(i915);
1929		mutex_unlock(&power_domains->lock);
1930		assert_isp_power_gated(i915);
1931	} else if (IS_VALLEYVIEW(i915)) {
1932		mutex_lock(&power_domains->lock);
1933		vlv_cmnlane_wa(i915);
1934		mutex_unlock(&power_domains->lock);
1935		assert_ved_power_gated(i915);
1936		assert_isp_power_gated(i915);
1937	} else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
1938		hsw_assert_cdclk(i915);
1939		intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1940	} else if (IS_IVYBRIDGE(i915)) {
1941		intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
1942	}
1943
1944	/*
1945	 * Keep all power wells enabled for any dependent HW access during
1946	 * initialization and to make sure we keep BIOS enabled display HW
1947	 * resources powered until display HW readout is complete. We drop
1948	 * this reference in intel_power_domains_enable().
1949	 */
1950	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
1951	power_domains->init_wakeref =
1952		intel_display_power_get(i915, POWER_DOMAIN_INIT);
1953
1954	/* Disable power support if the user asked so. */
1955	if (!i915->display.params.disable_power_well) {
1956		drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
1957		i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
1958										      POWER_DOMAIN_INIT);
1959	}
1960	intel_power_domains_sync_hw(i915);
1961
1962	power_domains->initializing = false;
1963}
1964
1965/**
1966 * intel_power_domains_driver_remove - deinitialize hw power domain state
1967 * @i915: i915 device instance
1968 *
1969 * De-initializes the display power domain HW state. It also ensures that the
1970 * device stays powered up so that the driver can be reloaded.
1971 *
1972 * It must be called with power domains already disabled (after a call to
1973 * intel_power_domains_disable()) and must be paired with
1974 * intel_power_domains_init_hw().
1975 */
1976void intel_power_domains_driver_remove(struct drm_i915_private *i915)
1977{
1978	intel_wakeref_t wakeref __maybe_unused =
1979		fetch_and_zero(&i915->display.power.domains.init_wakeref);
1980
1981	/* Remove the refcount we took to keep power well support disabled. */
1982	if (!i915->display.params.disable_power_well)
1983		intel_display_power_put(i915, POWER_DOMAIN_INIT,
1984					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
1985
1986	intel_display_power_flush_work_sync(i915);
1987
1988	intel_power_domains_verify_state(i915);
1989
1990	/* Keep the power well enabled, but cancel its rpm wakeref. */
1991	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1992}
1993
1994/**
1995 * intel_power_domains_sanitize_state - sanitize power domains state
1996 * @i915: i915 device instance
1997 *
1998 * Sanitize the power domains state during driver loading and system resume.
1999 * The function will disable all display power wells that BIOS has enabled
2000 * without a user for it (any user for a power well has taken a reference
2001 * on it by the time this function is called, after the state of all the
2002 * pipe, encoder, etc. HW resources have been sanitized).
2003 */
2004void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
2005{
2006	struct i915_power_domains *power_domains = &i915->display.power.domains;
2007	struct i915_power_well *power_well;
2008
2009	mutex_lock(&power_domains->lock);
2010
2011	for_each_power_well_reverse(i915, power_well) {
2012		if (power_well->desc->always_on || power_well->count ||
2013		    !intel_power_well_is_enabled(i915, power_well))
2014			continue;
2015
2016		drm_dbg_kms(&i915->drm,
2017			    "BIOS left unused %s power well enabled, disabling it\n",
2018			    intel_power_well_name(power_well));
2019		intel_power_well_disable(i915, power_well);
2020	}
2021
2022	mutex_unlock(&power_domains->lock);
2023}
2024
2025/**
2026 * intel_power_domains_enable - enable toggling of display power wells
2027 * @i915: i915 device instance
2028 *
2029 * Enable the ondemand enabling/disabling of the display power wells. Note that
2030 * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2031 * only at specific points of the display modeset sequence, thus they are not
2032 * affected by the intel_power_domains_enable()/disable() calls. The purpose
2033 * of these function is to keep the rest of power wells enabled until the end
2034 * of display HW readout (which will acquire the power references reflecting
2035 * the current HW state).
2036 */
2037void intel_power_domains_enable(struct drm_i915_private *i915)
2038{
2039	intel_wakeref_t wakeref __maybe_unused =
2040		fetch_and_zero(&i915->display.power.domains.init_wakeref);
2041
2042	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2043	intel_power_domains_verify_state(i915);
2044}
2045
2046/**
2047 * intel_power_domains_disable - disable toggling of display power wells
2048 * @i915: i915 device instance
2049 *
2050 * Disable the ondemand enabling/disabling of the display power wells. See
2051 * intel_power_domains_enable() for which power wells this call controls.
2052 */
2053void intel_power_domains_disable(struct drm_i915_private *i915)
2054{
2055	struct i915_power_domains *power_domains = &i915->display.power.domains;
2056
2057	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2058	power_domains->init_wakeref =
2059		intel_display_power_get(i915, POWER_DOMAIN_INIT);
2060
2061	intel_power_domains_verify_state(i915);
2062}
2063
2064/**
2065 * intel_power_domains_suspend - suspend power domain state
2066 * @i915: i915 device instance
2067 * @s2idle: specifies whether we go to idle, or deeper sleep
2068 *
2069 * This function prepares the hardware power domain state before entering
2070 * system suspend.
2071 *
2072 * It must be called with power domains already disabled (after a call to
2073 * intel_power_domains_disable()) and paired with intel_power_domains_resume().
2074 */
2075void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
2076{
2077	struct intel_display *display = &i915->display;
2078	struct i915_power_domains *power_domains = &display->power.domains;
2079	intel_wakeref_t wakeref __maybe_unused =
2080		fetch_and_zero(&power_domains->init_wakeref);
2081
2082	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
2083
2084	/*
2085	 * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2086	 * support don't manually deinit the power domains. This also means the
2087	 * DMC firmware will stay active, it will power down any HW
2088	 * resources as required and also enable deeper system power states
2089	 * that would be blocked if the firmware was inactive.
2090	 */
2091	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2092	    intel_dmc_has_payload(display)) {
2093		intel_display_power_flush_work(i915);
2094		intel_power_domains_verify_state(i915);
2095		return;
2096	}
2097
2098	/*
2099	 * Even if power well support was disabled we still want to disable
2100	 * power wells if power domains must be deinitialized for suspend.
2101	 */
2102	if (!i915->display.params.disable_power_well)
2103		intel_display_power_put(i915, POWER_DOMAIN_INIT,
2104					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
2105
2106	intel_display_power_flush_work(i915);
2107	intel_power_domains_verify_state(i915);
2108
2109	if (DISPLAY_VER(i915) >= 11)
2110		icl_display_core_uninit(i915);
2111	else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
2112		bxt_display_core_uninit(i915);
2113	else if (DISPLAY_VER(i915) == 9)
2114		skl_display_core_uninit(i915);
2115
2116	power_domains->display_core_suspended = true;
2117}
2118
2119/**
2120 * intel_power_domains_resume - resume power domain state
2121 * @i915: i915 device instance
2122 *
2123 * This function resume the hardware power domain state during system resume.
2124 *
2125 * It will return with power domain support disabled (to be enabled later by
2126 * intel_power_domains_enable()) and must be paired with
2127 * intel_power_domains_suspend().
2128 */
2129void intel_power_domains_resume(struct drm_i915_private *i915)
2130{
2131	struct i915_power_domains *power_domains = &i915->display.power.domains;
2132
2133	if (power_domains->display_core_suspended) {
2134		intel_power_domains_init_hw(i915, true);
2135		power_domains->display_core_suspended = false;
2136	} else {
2137		drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2138		power_domains->init_wakeref =
2139			intel_display_power_get(i915, POWER_DOMAIN_INIT);
2140	}
2141
2142	intel_power_domains_verify_state(i915);
2143}
2144
2145#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2146
2147static void intel_power_domains_dump_info(struct drm_i915_private *i915)
2148{
2149	struct i915_power_domains *power_domains = &i915->display.power.domains;
2150	struct i915_power_well *power_well;
2151
2152	for_each_power_well(i915, power_well) {
2153		enum intel_display_power_domain domain;
2154
2155		drm_dbg(&i915->drm, "%-25s %d\n",
2156			intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2157
2158		for_each_power_domain(domain, intel_power_well_domains(power_well))
2159			drm_dbg(&i915->drm, "  %-23s %d\n",
2160				intel_display_power_domain_str(domain),
2161				power_domains->domain_use_count[domain]);
2162	}
2163}
2164
2165/**
2166 * intel_power_domains_verify_state - verify the HW/SW state for all power wells
2167 * @i915: i915 device instance
2168 *
2169 * Verify if the reference count of each power well matches its HW enabled
2170 * state and the total refcount of the domains it belongs to. This must be
2171 * called after modeset HW state sanitization, which is responsible for
2172 * acquiring reference counts for any power wells in use and disabling the
2173 * ones left on by BIOS but not required by any active output.
2174 */
2175static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2176{
2177	struct i915_power_domains *power_domains = &i915->display.power.domains;
2178	struct i915_power_well *power_well;
2179	bool dump_domain_info;
2180
2181	mutex_lock(&power_domains->lock);
2182
2183	verify_async_put_domains_state(power_domains);
2184
2185	dump_domain_info = false;
2186	for_each_power_well(i915, power_well) {
2187		enum intel_display_power_domain domain;
2188		int domains_count;
2189		bool enabled;
2190
2191		enabled = intel_power_well_is_enabled(i915, power_well);
2192		if ((intel_power_well_refcount(power_well) ||
2193		     intel_power_well_is_always_on(power_well)) !=
2194		    enabled)
2195			drm_err(&i915->drm,
2196				"power well %s state mismatch (refcount %d/enabled %d)",
2197				intel_power_well_name(power_well),
2198				intel_power_well_refcount(power_well), enabled);
2199
2200		domains_count = 0;
2201		for_each_power_domain(domain, intel_power_well_domains(power_well))
2202			domains_count += power_domains->domain_use_count[domain];
2203
2204		if (intel_power_well_refcount(power_well) != domains_count) {
2205			drm_err(&i915->drm,
2206				"power well %s refcount/domain refcount mismatch "
2207				"(refcount %d/domains refcount %d)\n",
2208				intel_power_well_name(power_well),
2209				intel_power_well_refcount(power_well),
2210				domains_count);
2211			dump_domain_info = true;
2212		}
2213	}
2214
2215	if (dump_domain_info) {
2216		static bool dumped;
2217
2218		if (!dumped) {
2219			intel_power_domains_dump_info(i915);
2220			dumped = true;
2221		}
2222	}
2223
2224	mutex_unlock(&power_domains->lock);
2225}
2226
2227#else
2228
2229static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2230{
2231}
2232
2233#endif
2234
2235void intel_display_power_suspend_late(struct drm_i915_private *i915)
2236{
2237	struct intel_display *display = &i915->display;
2238
2239	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2240	    IS_BROXTON(i915)) {
2241		bxt_enable_dc9(display);
2242	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2243		hsw_enable_pc8(i915);
2244	}
2245
2246	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2247	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2248		intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2249}
2250
2251void intel_display_power_resume_early(struct drm_i915_private *i915)
2252{
2253	struct intel_display *display = &i915->display;
2254
2255	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2256	    IS_BROXTON(i915)) {
2257		gen9_sanitize_dc_state(display);
2258		bxt_disable_dc9(display);
2259	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2260		hsw_disable_pc8(i915);
2261	}
2262
2263	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2264	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2265		intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
2266}
2267
2268void intel_display_power_suspend(struct drm_i915_private *i915)
2269{
2270	struct intel_display *display = &i915->display;
2271
2272	if (DISPLAY_VER(i915) >= 11) {
2273		icl_display_core_uninit(i915);
2274		bxt_enable_dc9(display);
2275	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2276		bxt_display_core_uninit(i915);
2277		bxt_enable_dc9(display);
2278	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2279		hsw_enable_pc8(i915);
2280	}
2281}
2282
2283void intel_display_power_resume(struct drm_i915_private *i915)
2284{
2285	struct intel_display *display = &i915->display;
2286	struct i915_power_domains *power_domains = &display->power.domains;
2287
2288	if (DISPLAY_VER(i915) >= 11) {
2289		bxt_disable_dc9(display);
2290		icl_display_core_init(i915, true);
2291		if (intel_dmc_has_payload(display)) {
2292			if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2293				skl_enable_dc6(display);
2294			else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2295				gen9_enable_dc5(display);
2296		}
2297	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2298		bxt_disable_dc9(display);
2299		bxt_display_core_init(i915, true);
2300		if (intel_dmc_has_payload(display) &&
2301		    (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2302			gen9_enable_dc5(display);
2303	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2304		hsw_disable_pc8(i915);
2305	}
2306}
2307
2308void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
2309{
2310	struct i915_power_domains *power_domains = &i915->display.power.domains;
2311	int i;
2312
2313	mutex_lock(&power_domains->lock);
2314
2315	seq_printf(m, "%-25s %s\n", "Power well/domain", "Use count");
2316	for (i = 0; i < power_domains->power_well_count; i++) {
2317		struct i915_power_well *power_well;
2318		enum intel_display_power_domain power_domain;
2319
2320		power_well = &power_domains->power_wells[i];
2321		seq_printf(m, "%-25s %d\n", intel_power_well_name(power_well),
2322			   intel_power_well_refcount(power_well));
2323
2324		for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2325			seq_printf(m, "  %-23s %d\n",
2326				   intel_display_power_domain_str(power_domain),
2327				   power_domains->domain_use_count[power_domain]);
2328	}
2329
2330	mutex_unlock(&power_domains->lock);
2331}
2332
2333struct intel_ddi_port_domains {
2334	enum port port_start;
2335	enum port port_end;
2336	enum aux_ch aux_ch_start;
2337	enum aux_ch aux_ch_end;
2338
2339	enum intel_display_power_domain ddi_lanes;
2340	enum intel_display_power_domain ddi_io;
2341	enum intel_display_power_domain aux_io;
2342	enum intel_display_power_domain aux_legacy_usbc;
2343	enum intel_display_power_domain aux_tbt;
2344};
2345
2346static const struct intel_ddi_port_domains
2347i9xx_port_domains[] = {
2348	{
2349		.port_start = PORT_A,
2350		.port_end = PORT_F,
2351		.aux_ch_start = AUX_CH_A,
2352		.aux_ch_end = AUX_CH_F,
2353
2354		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2355		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2356		.aux_io = POWER_DOMAIN_AUX_IO_A,
2357		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2358		.aux_tbt = POWER_DOMAIN_INVALID,
2359	},
2360};
2361
2362static const struct intel_ddi_port_domains
2363d11_port_domains[] = {
2364	{
2365		.port_start = PORT_A,
2366		.port_end = PORT_B,
2367		.aux_ch_start = AUX_CH_A,
2368		.aux_ch_end = AUX_CH_B,
2369
2370		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2371		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2372		.aux_io = POWER_DOMAIN_AUX_IO_A,
2373		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2374		.aux_tbt = POWER_DOMAIN_INVALID,
2375	}, {
2376		.port_start = PORT_C,
2377		.port_end = PORT_F,
2378		.aux_ch_start = AUX_CH_C,
2379		.aux_ch_end = AUX_CH_F,
2380
2381		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2382		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2383		.aux_io = POWER_DOMAIN_AUX_IO_C,
2384		.aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2385		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2386	},
2387};
2388
2389static const struct intel_ddi_port_domains
2390d12_port_domains[] = {
2391	{
2392		.port_start = PORT_A,
2393		.port_end = PORT_C,
2394		.aux_ch_start = AUX_CH_A,
2395		.aux_ch_end = AUX_CH_C,
2396
2397		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2398		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2399		.aux_io = POWER_DOMAIN_AUX_IO_A,
2400		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2401		.aux_tbt = POWER_DOMAIN_INVALID,
2402	}, {
2403		.port_start = PORT_TC1,
2404		.port_end = PORT_TC6,
2405		.aux_ch_start = AUX_CH_USBC1,
2406		.aux_ch_end = AUX_CH_USBC6,
2407
2408		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2409		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2410		.aux_io = POWER_DOMAIN_INVALID,
2411		.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2412		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2413	},
2414};
2415
2416static const struct intel_ddi_port_domains
2417d13_port_domains[] = {
2418	{
2419		.port_start = PORT_A,
2420		.port_end = PORT_C,
2421		.aux_ch_start = AUX_CH_A,
2422		.aux_ch_end = AUX_CH_C,
2423
2424		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2425		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2426		.aux_io = POWER_DOMAIN_AUX_IO_A,
2427		.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2428		.aux_tbt = POWER_DOMAIN_INVALID,
2429	}, {
2430		.port_start = PORT_TC1,
2431		.port_end = PORT_TC4,
2432		.aux_ch_start = AUX_CH_USBC1,
2433		.aux_ch_end = AUX_CH_USBC4,
2434
2435		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2436		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2437		.aux_io = POWER_DOMAIN_INVALID,
2438		.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2439		.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2440	}, {
2441		.port_start = PORT_D_XELPD,
2442		.port_end = PORT_E_XELPD,
2443		.aux_ch_start = AUX_CH_D_XELPD,
2444		.aux_ch_end = AUX_CH_E_XELPD,
2445
2446		.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2447		.ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2448		.aux_io = POWER_DOMAIN_AUX_IO_D,
2449		.aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2450		.aux_tbt = POWER_DOMAIN_INVALID,
2451	},
2452};
2453
2454static void
2455intel_port_domains_for_platform(struct drm_i915_private *i915,
2456				const struct intel_ddi_port_domains **domains,
2457				int *domains_size)
2458{
2459	if (DISPLAY_VER(i915) >= 13) {
2460		*domains = d13_port_domains;
2461		*domains_size = ARRAY_SIZE(d13_port_domains);
2462	} else if (DISPLAY_VER(i915) >= 12) {
2463		*domains = d12_port_domains;
2464		*domains_size = ARRAY_SIZE(d12_port_domains);
2465	} else if (DISPLAY_VER(i915) >= 11) {
2466		*domains = d11_port_domains;
2467		*domains_size = ARRAY_SIZE(d11_port_domains);
2468	} else {
2469		*domains = i9xx_port_domains;
2470		*domains_size = ARRAY_SIZE(i9xx_port_domains);
2471	}
2472}
2473
2474static const struct intel_ddi_port_domains *
2475intel_port_domains_for_port(struct drm_i915_private *i915, enum port port)
2476{
2477	const struct intel_ddi_port_domains *domains;
2478	int domains_size;
2479	int i;
2480
2481	intel_port_domains_for_platform(i915, &domains, &domains_size);
2482	for (i = 0; i < domains_size; i++)
2483		if (port >= domains[i].port_start && port <= domains[i].port_end)
2484			return &domains[i];
2485
2486	return NULL;
2487}
2488
2489enum intel_display_power_domain
2490intel_display_power_ddi_io_domain(struct drm_i915_private *i915, enum port port)
2491{
2492	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2493
2494	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2495		return POWER_DOMAIN_PORT_DDI_IO_A;
2496
2497	return domains->ddi_io + (int)(port - domains->port_start);
2498}
2499
2500enum intel_display_power_domain
2501intel_display_power_ddi_lanes_domain(struct drm_i915_private *i915, enum port port)
2502{
2503	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2504
2505	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2506		return POWER_DOMAIN_PORT_DDI_LANES_A;
2507
2508	return domains->ddi_lanes + (int)(port - domains->port_start);
2509}
2510
2511static const struct intel_ddi_port_domains *
2512intel_port_domains_for_aux_ch(struct drm_i915_private *i915, enum aux_ch aux_ch)
2513{
2514	const struct intel_ddi_port_domains *domains;
2515	int domains_size;
2516	int i;
2517
2518	intel_port_domains_for_platform(i915, &domains, &domains_size);
2519	for (i = 0; i < domains_size; i++)
2520		if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2521			return &domains[i];
2522
2523	return NULL;
2524}
2525
2526enum intel_display_power_domain
2527intel_display_power_aux_io_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2528{
2529	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2530
2531	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2532		return POWER_DOMAIN_AUX_IO_A;
2533
2534	return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2535}
2536
2537enum intel_display_power_domain
2538intel_display_power_legacy_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2539{
2540	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2541
2542	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2543		return POWER_DOMAIN_AUX_A;
2544
2545	return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2546}
2547
2548enum intel_display_power_domain
2549intel_display_power_tbt_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2550{
2551	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2552
2553	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2554		return POWER_DOMAIN_AUX_TBT1;
2555
2556	return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);
2557}