1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2022 Intel Corporation
   4 */
   5
   6#include "i915_drv.h"
   7#include "i915_irq.h"
   8#include "i915_reg.h"
   9#include "intel_backlight_regs.h"
  10#include "intel_combo_phy.h"
  11#include "intel_combo_phy_regs.h"
  12#include "intel_crt.h"
  13#include "intel_de.h"
  14#include "intel_display_irq.h"
  15#include "intel_display_power_well.h"
  16#include "intel_display_types.h"
  17#include "intel_dkl_phy.h"
  18#include "intel_dkl_phy_regs.h"
  19#include "intel_dmc.h"
  20#include "intel_dmc_wl.h"
  21#include "intel_dp_aux_regs.h"
  22#include "intel_dpio_phy.h"
  23#include "intel_dpll.h"
  24#include "intel_hotplug.h"
  25#include "intel_pcode.h"
  26#include "intel_pps.h"
  27#include "intel_tc.h"
  28#include "intel_vga.h"
  29#include "skl_watermark.h"
  30#include "vlv_dpio_phy_regs.h"
  31#include "vlv_sideband.h"
  32#include "vlv_sideband_reg.h"
  33
  34struct i915_power_well_regs {
  35	i915_reg_t bios;
  36	i915_reg_t driver;
  37	i915_reg_t kvmr;
  38	i915_reg_t debug;
  39};
  40
  41struct i915_power_well_ops {
  42	const struct i915_power_well_regs *regs;
  43	/*
  44	 * Synchronize the well's hw state to match the current sw state, for
  45	 * example enable/disable it based on the current refcount. Called
  46	 * during driver init and resume time, possibly after first calling
  47	 * the enable/disable handlers.
  48	 */
  49	void (*sync_hw)(struct drm_i915_private *i915,
  50			struct i915_power_well *power_well);
  51	/*
  52	 * Enable the well and resources that depend on it (for example
  53	 * interrupts located on the well). Called after the 0->1 refcount
  54	 * transition.
  55	 */
  56	void (*enable)(struct drm_i915_private *i915,
  57		       struct i915_power_well *power_well);
  58	/*
  59	 * Disable the well and resources that depend on it. Called after
  60	 * the 1->0 refcount transition.
  61	 */
  62	void (*disable)(struct drm_i915_private *i915,
  63			struct i915_power_well *power_well);
  64	/* Returns the hw enabled state. */
  65	bool (*is_enabled)(struct drm_i915_private *i915,
  66			   struct i915_power_well *power_well);
  67};
  68
  69static const struct i915_power_well_instance *
  70i915_power_well_instance(const struct i915_power_well *power_well)
  71{
  72	return &power_well->desc->instances->list[power_well->instance_idx];
  73}
  74
  75struct i915_power_well *
  76lookup_power_well(struct drm_i915_private *i915,
  77		  enum i915_power_well_id power_well_id)
  78{
  79	struct i915_power_well *power_well;
  80
  81	for_each_power_well(i915, power_well)
  82		if (i915_power_well_instance(power_well)->id == power_well_id)
  83			return power_well;
  84
  85	/*
  86	 * It's not feasible to add error checking code to the callers since
  87	 * this condition really shouldn't happen and it doesn't even make sense
  88	 * to abort things like display initialization sequences. Just return
  89	 * the first power well and hope the WARN gets reported so we can fix
  90	 * our driver.
  91	 */
  92	drm_WARN(&i915->drm, 1,
  93		 "Power well %d not defined for this platform\n",
  94		 power_well_id);
  95	return &i915->display.power.domains.power_wells[0];
  96}
  97
  98void intel_power_well_enable(struct drm_i915_private *i915,
  99			     struct i915_power_well *power_well)
 100{
 101	drm_dbg_kms(&i915->drm, "enabling %s\n", intel_power_well_name(power_well));
 102	power_well->desc->ops->enable(i915, power_well);
 103	power_well->hw_enabled = true;
 104}
 105
 106void intel_power_well_disable(struct drm_i915_private *i915,
 107			      struct i915_power_well *power_well)
 108{
 109	drm_dbg_kms(&i915->drm, "disabling %s\n", intel_power_well_name(power_well));
 110	power_well->hw_enabled = false;
 111	power_well->desc->ops->disable(i915, power_well);
 112}
 113
 114void intel_power_well_sync_hw(struct drm_i915_private *i915,
 115			      struct i915_power_well *power_well)
 116{
 117	power_well->desc->ops->sync_hw(i915, power_well);
 118	power_well->hw_enabled =
 119		power_well->desc->ops->is_enabled(i915, power_well);
 120}
 121
 122void intel_power_well_get(struct drm_i915_private *i915,
 123			  struct i915_power_well *power_well)
 124{
 125	if (!power_well->count++)
 126		intel_power_well_enable(i915, power_well);
 127}
 128
 129void intel_power_well_put(struct drm_i915_private *i915,
 130			  struct i915_power_well *power_well)
 131{
 132	drm_WARN(&i915->drm, !power_well->count,
 133		 "Use count on power well %s is already zero",
 134		 i915_power_well_instance(power_well)->name);
 135
 136	if (!--power_well->count)
 137		intel_power_well_disable(i915, power_well);
 138}
 139
 140bool intel_power_well_is_enabled(struct drm_i915_private *i915,
 141				 struct i915_power_well *power_well)
 142{
 143	return power_well->desc->ops->is_enabled(i915, power_well);
 144}
 145
 146bool intel_power_well_is_enabled_cached(struct i915_power_well *power_well)
 147{
 148	return power_well->hw_enabled;
 149}
 150
 151bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
 152					 enum i915_power_well_id power_well_id)
 153{
 154	struct i915_power_well *power_well;
 155
 156	power_well = lookup_power_well(dev_priv, power_well_id);
 157
 158	return intel_power_well_is_enabled(dev_priv, power_well);
 159}
 160
 161bool intel_power_well_is_always_on(struct i915_power_well *power_well)
 162{
 163	return power_well->desc->always_on;
 164}
 165
 166const char *intel_power_well_name(struct i915_power_well *power_well)
 167{
 168	return i915_power_well_instance(power_well)->name;
 169}
 170
 171struct intel_power_domain_mask *intel_power_well_domains(struct i915_power_well *power_well)
 172{
 173	return &power_well->domains;
 174}
 175
 176int intel_power_well_refcount(struct i915_power_well *power_well)
 177{
 178	return power_well->count;
 179}
 180
 181/*
 182 * Starting with Haswell, we have a "Power Down Well" that can be turned off
 183 * when not needed anymore. We have 4 registers that can request the power well
 184 * to be enabled, and it will only be disabled if none of the registers is
 185 * requesting it to be enabled.
 186 */
 187static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 188				       u8 irq_pipe_mask, bool has_vga)
 189{
 190	struct intel_display *display = &dev_priv->display;
 191
 192	if (has_vga)
 193		intel_vga_reset_io_mem(display);
 194
 195	if (irq_pipe_mask)
 196		gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 197}
 198
 199static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 200				       u8 irq_pipe_mask)
 201{
 202	if (irq_pipe_mask)
 203		gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 204}
 205
 206#define ICL_AUX_PW_TO_PHY(pw_idx)	\
 207	((pw_idx) - ICL_PW_CTL_IDX_AUX_A + PHY_A)
 208
 209#define ICL_AUX_PW_TO_CH(pw_idx)	\
 210	((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 211
 212#define ICL_TBT_AUX_PW_TO_CH(pw_idx)	\
 213	((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
 214
 215static enum aux_ch icl_aux_pw_to_ch(const struct i915_power_well *power_well)
 216{
 217	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 218
 219	return power_well->desc->is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
 220					     ICL_AUX_PW_TO_CH(pw_idx);
 221}
 222
 223static struct intel_digital_port *
 224aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
 225		       enum aux_ch aux_ch)
 226{
 
 227	struct intel_encoder *encoder;
 228
 229	for_each_intel_encoder(&dev_priv->drm, encoder) {
 230		struct intel_digital_port *dig_port;
 231
 232		/* We'll check the MST primary port */
 233		if (encoder->type == INTEL_OUTPUT_DP_MST)
 234			continue;
 235
 236		dig_port = enc_to_dig_port(encoder);
 
 
 237
 238		if (dig_port && dig_port->aux_ch == aux_ch)
 239			return dig_port;
 
 
 
 
 240	}
 241
 242	return NULL;
 243}
 244
 245static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915,
 246				  const struct i915_power_well *power_well)
 247{
 248	enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 249	struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch);
 250
 251	/*
 252	 * FIXME should we care about the (VBT defined) dig_port->aux_ch
 253	 * relationship or should this be purely defined by the hardware layout?
 254	 * Currently if the port doesn't appear in the VBT, or if it's declared
 255	 * as HDMI-only and routed to a combo PHY, the encoder either won't be
 256	 * present at all or it will not have an aux_ch assigned.
 257	 */
 258	return dig_port ? intel_encoder_to_phy(&dig_port->base) : PHY_NONE;
 259}
 260
 261static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 262					   struct i915_power_well *power_well,
 263					   bool timeout_expected)
 264{
 265	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 266	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 267	int timeout = power_well->desc->enable_timeout ? : 1;
 268
 269	/*
 270	 * For some power wells we're not supposed to watch the status bit for
 271	 * an ack, but rather just wait a fixed amount of time and then
 272	 * proceed.  This is only used on DG2.
 273	 */
 274	if (IS_DG2(dev_priv) && power_well->desc->fixed_enable_delay) {
 275		usleep_range(600, 1200);
 276		return;
 277	}
 278
 279	/* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 280	if (intel_de_wait_for_set(dev_priv, regs->driver,
 281				  HSW_PWR_WELL_CTL_STATE(pw_idx), timeout)) {
 282		drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n",
 283			    intel_power_well_name(power_well));
 284
 285		drm_WARN_ON(&dev_priv->drm, !timeout_expected);
 286
 287	}
 288}
 289
 290static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 291				     const struct i915_power_well_regs *regs,
 292				     int pw_idx)
 293{
 294	u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 295	u32 ret;
 296
 297	ret = intel_de_read(dev_priv, regs->bios) & req_mask ? 1 : 0;
 298	ret |= intel_de_read(dev_priv, regs->driver) & req_mask ? 2 : 0;
 299	if (regs->kvmr.reg)
 300		ret |= intel_de_read(dev_priv, regs->kvmr) & req_mask ? 4 : 0;
 301	ret |= intel_de_read(dev_priv, regs->debug) & req_mask ? 8 : 0;
 302
 303	return ret;
 304}
 305
 306static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 307					    struct i915_power_well *power_well)
 308{
 309	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 310	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 311	bool disabled;
 312	u32 reqs;
 313
 314	/*
 315	 * Bspec doesn't require waiting for PWs to get disabled, but still do
 316	 * this for paranoia. The known cases where a PW will be forced on:
 317	 * - a KVMR request on any power well via the KVMR request register
 318	 * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 319	 *   DEBUG request registers
 320	 * Skip the wait in case any of the request bits are set and print a
 321	 * diagnostic message.
 322	 */
 323	wait_for((disabled = !(intel_de_read(dev_priv, regs->driver) &
 324			       HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 325		 (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
 326	if (disabled)
 327		return;
 328
 329	drm_dbg_kms(&dev_priv->drm,
 330		    "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 331		    intel_power_well_name(power_well),
 332		    !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 333}
 334
 335static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 336					   enum skl_power_gate pg)
 337{
 338	/* Timeout 5us for PG#0, for other PGs 1us */
 339	drm_WARN_ON(&dev_priv->drm,
 340		    intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS,
 341					  SKL_FUSE_PG_DIST_STATUS(pg), 1));
 342}
 343
 344static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 345				  struct i915_power_well *power_well)
 346{
 347	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 348	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 349
 350	if (power_well->desc->has_fuses) {
 351		enum skl_power_gate pg;
 352
 353		pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 354						 SKL_PW_CTL_IDX_TO_PG(pw_idx);
 355
 356		/* Wa_16013190616:adlp */
 357		if (IS_ALDERLAKE_P(dev_priv) && pg == SKL_PG1)
 358			intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, 0, DISABLE_FLR_SRC);
 359
 360		/*
 361		 * For PW1 we have to wait both for the PW0/PG0 fuse state
 362		 * before enabling the power well and PW1/PG1's own fuse
 363		 * state after the enabling. For all other power wells with
 364		 * fuses we only have to wait for that PW/PG's fuse state
 365		 * after the enabling.
 366		 */
 367		if (pg == SKL_PG1)
 368			gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 369	}
 370
 371	intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 372
 373	hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 374
 375	if (power_well->desc->has_fuses) {
 376		enum skl_power_gate pg;
 377
 378		pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 379						 SKL_PW_CTL_IDX_TO_PG(pw_idx);
 380		gen9_wait_for_power_well_fuses(dev_priv, pg);
 381	}
 382
 383	hsw_power_well_post_enable(dev_priv,
 384				   power_well->desc->irq_pipe_mask,
 385				   power_well->desc->has_vga);
 386}
 387
 388static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 389				   struct i915_power_well *power_well)
 390{
 391	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 392	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 393
 394	hsw_power_well_pre_disable(dev_priv,
 395				   power_well->desc->irq_pipe_mask);
 396
 397	intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 398	hsw_wait_for_power_well_disable(dev_priv, power_well);
 399}
 400
 401static bool intel_aux_ch_is_edp(struct drm_i915_private *i915, enum aux_ch aux_ch)
 402{
 403	struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch);
 404
 405	return dig_port && dig_port->base.type == INTEL_OUTPUT_EDP;
 
 
 
 
 
 
 406}
 407
 408static void
 409icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 410				    struct i915_power_well *power_well)
 411{
 412	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 413	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 
 414
 415	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 416
 417	intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 418
 419	/*
 420	 * FIXME not sure if we should derive the PHY from the pw_idx, or
 421	 * from the VBT defined AUX_CH->DDI->PHY mapping.
 422	 */
 423	intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
 424		     0, ICL_LANE_ENABLE_AUX);
 425
 426	hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 427
 428	/* Display WA #1178: icl */
 429	if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 430	    !intel_aux_ch_is_edp(dev_priv, ICL_AUX_PW_TO_CH(pw_idx)))
 431		intel_de_rmw(dev_priv, ICL_PORT_TX_DW6_AUX(ICL_AUX_PW_TO_PHY(pw_idx)),
 432			     0, O_FUNC_OVRD_EN | O_LDO_BYPASS_CRI);
 433}
 434
 435static void
 436icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 437				     struct i915_power_well *power_well)
 438{
 439	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 440	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 
 441
 442	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 443
 444	/*
 445	 * FIXME not sure if we should derive the PHY from the pw_idx, or
 446	 * from the VBT defined AUX_CH->DDI->PHY mapping.
 447	 */
 448	intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
 449		     ICL_LANE_ENABLE_AUX, 0);
 450
 451	intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 452
 453	hsw_wait_for_power_well_disable(dev_priv, power_well);
 454}
 455
 456#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 457
 458static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 459					struct i915_power_well *power_well,
 460					struct intel_digital_port *dig_port)
 461{
 462	if (drm_WARN_ON(&dev_priv->drm, !dig_port))
 463		return;
 464
 465	if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
 466		return;
 467
 468	drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port));
 469}
 470
 471#else
 472
 473static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 474					struct i915_power_well *power_well,
 475					struct intel_digital_port *dig_port)
 476{
 477}
 478
 479#endif
 480
 481#define TGL_AUX_PW_TO_TC_PORT(pw_idx)	((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
 482
 483static void icl_tc_cold_exit(struct drm_i915_private *i915)
 484{
 485	int ret, tries = 0;
 486
 487	while (1) {
 488		ret = snb_pcode_write_timeout(&i915->uncore, ICL_PCODE_EXIT_TCCOLD, 0,
 489					      250, 1);
 490		if (ret != -EAGAIN || ++tries == 3)
 491			break;
 492		msleep(1);
 493	}
 494
 495	/* Spec states that TC cold exit can take up to 1ms to complete */
 496	if (!ret)
 497		msleep(1);
 498
 499	/* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */
 500	drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" :
 501		    "succeeded");
 502}
 503
 504static void
 505icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 506				 struct i915_power_well *power_well)
 507{
 508	enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 509	struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 510	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 511	bool is_tbt = power_well->desc->is_tc_tbt;
 512	bool timeout_expected;
 513
 514	icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 515
 516	intel_de_rmw(dev_priv, DP_AUX_CH_CTL(aux_ch),
 517		     DP_AUX_CH_CTL_TBT_IO, is_tbt ? DP_AUX_CH_CTL_TBT_IO : 0);
 518
 519	intel_de_rmw(dev_priv, regs->driver,
 520		     0,
 521		     HSW_PWR_WELL_CTL_REQ(i915_power_well_instance(power_well)->hsw.idx));
 522
 523	/*
 524	 * An AUX timeout is expected if the TBT DP tunnel is down,
 525	 * or need to enable AUX on a legacy TypeC port as part of the TC-cold
 526	 * exit sequence.
 527	 */
 528	timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port);
 529	if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
 530		icl_tc_cold_exit(dev_priv);
 531
 532	hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected);
 533
 534	if (DISPLAY_VER(dev_priv) >= 12 && !is_tbt) {
 535		enum tc_port tc_port;
 536
 537		tc_port = TGL_AUX_PW_TO_TC_PORT(i915_power_well_instance(power_well)->hsw.idx);
 538
 539		if (wait_for(intel_dkl_phy_read(dev_priv, DKL_CMN_UC_DW_27(tc_port)) &
 540			     DKL_CMN_UC_DW27_UC_HEALTH, 1))
 541			drm_warn(&dev_priv->drm,
 542				 "Timeout waiting TC uC health\n");
 543	}
 544}
 545
 546static void
 547icl_aux_power_well_enable(struct drm_i915_private *dev_priv,
 548			  struct i915_power_well *power_well)
 549{
 550	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 551
 552	if (intel_phy_is_tc(dev_priv, phy))
 553		return icl_tc_phy_aux_power_well_enable(dev_priv, power_well);
 554	else if (IS_ICELAKE(dev_priv))
 555		return icl_combo_phy_aux_power_well_enable(dev_priv,
 556							   power_well);
 557	else
 558		return hsw_power_well_enable(dev_priv, power_well);
 559}
 560
 561static void
 562icl_aux_power_well_disable(struct drm_i915_private *dev_priv,
 563			   struct i915_power_well *power_well)
 564{
 565	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 566
 567	if (intel_phy_is_tc(dev_priv, phy))
 568		return hsw_power_well_disable(dev_priv, power_well);
 569	else if (IS_ICELAKE(dev_priv))
 570		return icl_combo_phy_aux_power_well_disable(dev_priv,
 571							    power_well);
 572	else
 573		return hsw_power_well_disable(dev_priv, power_well);
 574}
 575
 576/*
 577 * We should only use the power well if we explicitly asked the hardware to
 578 * enable it, so check if it's enabled and also check if we've requested it to
 579 * be enabled.
 580 */
 581static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 582				   struct i915_power_well *power_well)
 583{
 584	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 585	enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
 586	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 587	u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 588		   HSW_PWR_WELL_CTL_STATE(pw_idx);
 589	u32 val;
 590
 591	val = intel_de_read(dev_priv, regs->driver);
 592
 593	/*
 594	 * On GEN9 big core due to a DMC bug the driver's request bits for PW1
 595	 * and the MISC_IO PW will be not restored, so check instead for the
 596	 * BIOS's own request bits, which are forced-on for these power wells
 597	 * when exiting DC5/6.
 598	 */
 599	if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) &&
 600	    (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
 601		val |= intel_de_read(dev_priv, regs->bios);
 602
 603	return (val & mask) == mask;
 604}
 605
 606static void assert_can_enable_dc9(struct intel_display *display)
 607{
 608	struct drm_i915_private *dev_priv = to_i915(display->drm);
 609
 610	drm_WARN_ONCE(display->drm,
 611		      (intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC9),
 612		      "DC9 already programmed to be enabled.\n");
 613	drm_WARN_ONCE(display->drm,
 614		      intel_de_read(display, DC_STATE_EN) &
 615		      DC_STATE_EN_UPTO_DC5,
 616		      "DC5 still not disabled to enable DC9.\n");
 617	drm_WARN_ONCE(display->drm,
 618		      intel_de_read(display, HSW_PWR_WELL_CTL2) &
 619		      HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 620		      "Power well 2 on.\n");
 621	drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
 622		      "Interrupts not disabled yet.\n");
 623
 624	 /*
 625	  * TODO: check for the following to verify the conditions to enter DC9
 626	  * state are satisfied:
 627	  * 1] Check relevant display engine registers to verify if mode set
 628	  * disable sequence was followed.
 629	  * 2] Check if display uninitialize sequence is initialized.
 630	  */
 631}
 632
 633static void assert_can_disable_dc9(struct intel_display *display)
 634{
 635	struct drm_i915_private *dev_priv = to_i915(display->drm);
 636
 637	drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
 638		      "Interrupts not disabled yet.\n");
 639	drm_WARN_ONCE(display->drm,
 640		      intel_de_read(display, DC_STATE_EN) &
 641		      DC_STATE_EN_UPTO_DC5,
 642		      "DC5 still not disabled.\n");
 643
 644	 /*
 645	  * TODO: check for the following to verify DC9 state was indeed
 646	  * entered before programming to disable it:
 647	  * 1] Check relevant display engine registers to verify if mode
 648	  *  set disable sequence was followed.
 649	  * 2] Check if display uninitialize sequence is initialized.
 650	  */
 651}
 652
 653static void gen9_write_dc_state(struct intel_display *display,
 654				u32 state)
 655{
 656	int rewrites = 0;
 657	int rereads = 0;
 658	u32 v;
 659
 660	intel_de_write(display, DC_STATE_EN, state);
 661
 662	/* It has been observed that disabling the dc6 state sometimes
 663	 * doesn't stick and dmc keeps returning old value. Make sure
 664	 * the write really sticks enough times and also force rewrite until
 665	 * we are confident that state is exactly what we want.
 666	 */
 667	do  {
 668		v = intel_de_read(display, DC_STATE_EN);
 669
 670		if (v != state) {
 671			intel_de_write(display, DC_STATE_EN, state);
 672			rewrites++;
 673			rereads = 0;
 674		} else if (rereads++ > 5) {
 675			break;
 676		}
 677
 678	} while (rewrites < 100);
 679
 680	if (v != state)
 681		drm_err(display->drm,
 682			"Writing dc state to 0x%x failed, now 0x%x\n",
 683			state, v);
 684
 685	/* Most of the times we need one retry, avoid spam */
 686	if (rewrites > 1)
 687		drm_dbg_kms(display->drm,
 688			    "Rewrote dc state to 0x%x %d times\n",
 689			    state, rewrites);
 690}
 691
 692static u32 gen9_dc_mask(struct intel_display *display)
 693{
 694	struct drm_i915_private *dev_priv = to_i915(display->drm);
 695	u32 mask;
 696
 697	mask = DC_STATE_EN_UPTO_DC5;
 698
 699	if (DISPLAY_VER(display) >= 12)
 700		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
 701					  | DC_STATE_EN_DC9;
 702	else if (DISPLAY_VER(display) == 11)
 703		mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 704	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 705		mask |= DC_STATE_EN_DC9;
 706	else
 707		mask |= DC_STATE_EN_UPTO_DC6;
 708
 709	return mask;
 710}
 711
 712void gen9_sanitize_dc_state(struct intel_display *display)
 713{
 714	struct i915_power_domains *power_domains = &display->power.domains;
 715	u32 val;
 716
 717	if (!HAS_DISPLAY(display))
 718		return;
 719
 720	val = intel_de_read(display, DC_STATE_EN) & gen9_dc_mask(display);
 721
 722	drm_dbg_kms(display->drm,
 723		    "Resetting DC state tracking from %02x to %02x\n",
 724		    power_domains->dc_state, val);
 725	power_domains->dc_state = val;
 726}
 727
 728/**
 729 * gen9_set_dc_state - set target display C power state
 730 * @display: display instance
 731 * @state: target DC power state
 732 * - DC_STATE_DISABLE
 733 * - DC_STATE_EN_UPTO_DC5
 734 * - DC_STATE_EN_UPTO_DC6
 735 * - DC_STATE_EN_DC9
 736 *
 737 * Signal to DMC firmware/HW the target DC power state passed in @state.
 738 * DMC/HW can turn off individual display clocks and power rails when entering
 739 * a deeper DC power state (higher in number) and turns these back when exiting
 740 * that state to a shallower power state (lower in number). The HW will decide
 741 * when to actually enter a given state on an on-demand basis, for instance
 742 * depending on the active state of display pipes. The state of display
 743 * registers backed by affected power rails are saved/restored as needed.
 744 *
 745 * Based on the above enabling a deeper DC power state is asynchronous wrt.
 746 * enabling it. Disabling a deeper power state is synchronous: for instance
 747 * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 748 * back on and register state is restored. This is guaranteed by the MMIO write
 749 * to DC_STATE_EN blocking until the state is restored.
 750 */
 751void gen9_set_dc_state(struct intel_display *display, u32 state)
 752{
 753	struct i915_power_domains *power_domains = &display->power.domains;
 754	u32 val;
 755	u32 mask;
 756
 757	if (!HAS_DISPLAY(display))
 758		return;
 759
 760	if (drm_WARN_ON_ONCE(display->drm,
 761			     state & ~power_domains->allowed_dc_mask))
 762		state &= power_domains->allowed_dc_mask;
 763
 764	val = intel_de_read(display, DC_STATE_EN);
 765	mask = gen9_dc_mask(display);
 766	drm_dbg_kms(display->drm, "Setting DC state from %02x to %02x\n",
 767		    val & mask, state);
 768
 769	/* Check if DMC is ignoring our DC state requests */
 770	if ((val & mask) != power_domains->dc_state)
 771		drm_err(display->drm, "DC state mismatch (0x%x -> 0x%x)\n",
 772			power_domains->dc_state, val & mask);
 773
 774	val &= ~mask;
 775	val |= state;
 776
 777	gen9_write_dc_state(display, val);
 778
 779	power_domains->dc_state = val & mask;
 780}
 781
 782static void tgl_enable_dc3co(struct intel_display *display)
 783{
 784	drm_dbg_kms(display->drm, "Enabling DC3CO\n");
 785	gen9_set_dc_state(display, DC_STATE_EN_DC3CO);
 786}
 787
 788static void tgl_disable_dc3co(struct intel_display *display)
 789{
 790	drm_dbg_kms(display->drm, "Disabling DC3CO\n");
 791	intel_de_rmw(display, DC_STATE_EN, DC_STATE_DC3CO_STATUS, 0);
 792	gen9_set_dc_state(display, DC_STATE_DISABLE);
 793	/*
 794	 * Delay of 200us DC3CO Exit time B.Spec 49196
 795	 */
 796	usleep_range(200, 210);
 797}
 798
 799static void assert_can_enable_dc5(struct intel_display *display)
 800{
 801	struct drm_i915_private *dev_priv = to_i915(display->drm);
 802	enum i915_power_well_id high_pg;
 803
 804	/* Power wells at this level and above must be disabled for DC5 entry */
 805	if (DISPLAY_VER(display) == 12)
 806		high_pg = ICL_DISP_PW_3;
 807	else
 808		high_pg = SKL_DISP_PW_2;
 809
 810	drm_WARN_ONCE(display->drm,
 811		      intel_display_power_well_is_enabled(dev_priv, high_pg),
 812		      "Power wells above platform's DC5 limit still enabled.\n");
 813
 814	drm_WARN_ONCE(display->drm,
 815		      (intel_de_read(display, DC_STATE_EN) &
 816		       DC_STATE_EN_UPTO_DC5),
 817		      "DC5 already programmed to be enabled.\n");
 818	assert_rpm_wakelock_held(&dev_priv->runtime_pm);
 819
 820	assert_dmc_loaded(display);
 821}
 822
 823void gen9_enable_dc5(struct intel_display *display)
 824{
 825	struct drm_i915_private *dev_priv = to_i915(display->drm);
 826
 827	assert_can_enable_dc5(display);
 828
 829	drm_dbg_kms(display->drm, "Enabling DC5\n");
 830
 831	/* Wa Display #1183: skl,kbl,cfl */
 832	if (DISPLAY_VER(display) == 9 && !IS_BROXTON(dev_priv))
 833		intel_de_rmw(display, GEN8_CHICKEN_DCPR_1,
 834			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 835
 836	intel_dmc_wl_enable(display);
 837
 838	gen9_set_dc_state(display, DC_STATE_EN_UPTO_DC5);
 839}
 840
 841static void assert_can_enable_dc6(struct intel_display *display)
 842{
 843	drm_WARN_ONCE(display->drm,
 844		      (intel_de_read(display, UTIL_PIN_CTL) &
 845		       (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) ==
 846		      (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
 847		      "Utility pin enabled in PWM mode\n");
 848	drm_WARN_ONCE(display->drm,
 849		      (intel_de_read(display, DC_STATE_EN) &
 850		       DC_STATE_EN_UPTO_DC6),
 851		      "DC6 already programmed to be enabled.\n");
 852
 853	assert_dmc_loaded(display);
 854}
 855
 856void skl_enable_dc6(struct intel_display *display)
 857{
 858	struct drm_i915_private *dev_priv = to_i915(display->drm);
 859
 860	assert_can_enable_dc6(display);
 861
 862	drm_dbg_kms(display->drm, "Enabling DC6\n");
 863
 864	/* Wa Display #1183: skl,kbl,cfl */
 865	if (DISPLAY_VER(display) == 9 && !IS_BROXTON(dev_priv))
 866		intel_de_rmw(display, GEN8_CHICKEN_DCPR_1,
 867			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 868
 869	intel_dmc_wl_enable(display);
 870
 871	gen9_set_dc_state(display, DC_STATE_EN_UPTO_DC6);
 872}
 873
 874void bxt_enable_dc9(struct intel_display *display)
 875{
 876	struct drm_i915_private *dev_priv = to_i915(display->drm);
 877
 878	assert_can_enable_dc9(display);
 879
 880	drm_dbg_kms(display->drm, "Enabling DC9\n");
 881	/*
 882	 * Power sequencer reset is needed on BXT/GLK, because the PPS registers
 883	 * aren't always on, unlike with South Display Engine on PCH.
 
 884	 */
 885	if (IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv))
 886		bxt_pps_reset_all(display);
 887	gen9_set_dc_state(display, DC_STATE_EN_DC9);
 888}
 889
 890void bxt_disable_dc9(struct intel_display *display)
 891{
 892	assert_can_disable_dc9(display);
 893
 894	drm_dbg_kms(display->drm, "Disabling DC9\n");
 895
 896	gen9_set_dc_state(display, DC_STATE_DISABLE);
 897
 898	intel_pps_unlock_regs_wa(display);
 899}
 900
 901static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 902				   struct i915_power_well *power_well)
 903{
 904	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 905	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 906	u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 907	u32 bios_req = intel_de_read(dev_priv, regs->bios);
 908
 909	/* Take over the request bit if set by BIOS. */
 910	if (bios_req & mask) {
 911		u32 drv_req = intel_de_read(dev_priv, regs->driver);
 912
 913		if (!(drv_req & mask))
 914			intel_de_write(dev_priv, regs->driver, drv_req | mask);
 915		intel_de_write(dev_priv, regs->bios, bios_req & ~mask);
 916	}
 917}
 918
 919static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 920					   struct i915_power_well *power_well)
 921{
 922	struct intel_display *display = &dev_priv->display;
 923
 924	bxt_dpio_phy_init(display, i915_power_well_instance(power_well)->bxt.phy);
 925}
 926
 927static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 928					    struct i915_power_well *power_well)
 929{
 930	struct intel_display *display = &dev_priv->display;
 931
 932	bxt_dpio_phy_uninit(display, i915_power_well_instance(power_well)->bxt.phy);
 933}
 934
 935static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 936					    struct i915_power_well *power_well)
 937{
 938	struct intel_display *display = &dev_priv->display;
 939
 940	return bxt_dpio_phy_is_enabled(display, i915_power_well_instance(power_well)->bxt.phy);
 941}
 942
 943static void bxt_verify_dpio_phy_power_wells(struct drm_i915_private *dev_priv)
 944{
 945	struct intel_display *display = &dev_priv->display;
 946	struct i915_power_well *power_well;
 947
 948	power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
 949	if (intel_power_well_refcount(power_well) > 0)
 950		bxt_dpio_phy_verify_state(display, i915_power_well_instance(power_well)->bxt.phy);
 951
 952	power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
 953	if (intel_power_well_refcount(power_well) > 0)
 954		bxt_dpio_phy_verify_state(display, i915_power_well_instance(power_well)->bxt.phy);
 955
 956	if (IS_GEMINILAKE(dev_priv)) {
 957		power_well = lookup_power_well(dev_priv,
 958					       GLK_DISP_PW_DPIO_CMN_C);
 959		if (intel_power_well_refcount(power_well) > 0)
 960			bxt_dpio_phy_verify_state(display,
 961						  i915_power_well_instance(power_well)->bxt.phy);
 962	}
 963}
 964
 965static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 966					   struct i915_power_well *power_well)
 967{
 968	struct intel_display *display = &dev_priv->display;
 969
 970	return ((intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
 971		(intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
 972}
 973
 974static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 975{
 976	u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
 977	u8 enabled_dbuf_slices = dev_priv->display.dbuf.enabled_slices;
 978
 979	drm_WARN(&dev_priv->drm,
 980		 hw_enabled_dbuf_slices != enabled_dbuf_slices,
 981		 "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
 982		 hw_enabled_dbuf_slices,
 983		 enabled_dbuf_slices);
 984}
 985
 986void gen9_disable_dc_states(struct intel_display *display)
 987{
 988	struct drm_i915_private *dev_priv = to_i915(display->drm);
 989	struct i915_power_domains *power_domains = &display->power.domains;
 990	struct intel_cdclk_config cdclk_config = {};
 991
 992	if (power_domains->target_dc_state == DC_STATE_EN_DC3CO) {
 993		tgl_disable_dc3co(display);
 994		return;
 995	}
 996
 997	gen9_set_dc_state(display, DC_STATE_DISABLE);
 998
 999	if (!HAS_DISPLAY(display))
1000		return;
1001
1002	intel_dmc_wl_disable(display);
1003
1004	intel_cdclk_get_cdclk(display, &cdclk_config);
1005	/* Can't read out voltage_level so can't use intel_cdclk_changed() */
1006	drm_WARN_ON(display->drm,
1007		    intel_cdclk_clock_changed(&display->cdclk.hw,
1008					      &cdclk_config));
1009
1010	gen9_assert_dbuf_enabled(dev_priv);
1011
1012	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
1013		bxt_verify_dpio_phy_power_wells(dev_priv);
1014
1015	if (DISPLAY_VER(display) >= 11)
1016		/*
1017		 * DMC retains HW context only for port A, the other combo
1018		 * PHY's HW context for port B is lost after DC transitions,
1019		 * so we need to restore it manually.
1020		 */
1021		intel_combo_phy_init(dev_priv);
1022}
1023
1024static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
1025					  struct i915_power_well *power_well)
1026{
1027	struct intel_display *display = &dev_priv->display;
1028
1029	gen9_disable_dc_states(display);
1030}
1031
1032static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
1033					   struct i915_power_well *power_well)
1034{
1035	struct intel_display *display = &dev_priv->display;
1036	struct i915_power_domains *power_domains = &display->power.domains;
1037
1038	if (!intel_dmc_has_payload(display))
1039		return;
1040
1041	switch (power_domains->target_dc_state) {
1042	case DC_STATE_EN_DC3CO:
1043		tgl_enable_dc3co(display);
1044		break;
1045	case DC_STATE_EN_UPTO_DC6:
1046		skl_enable_dc6(display);
1047		break;
1048	case DC_STATE_EN_UPTO_DC5:
1049		gen9_enable_dc5(display);
1050		break;
1051	}
1052}
1053
1054static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
1055					 struct i915_power_well *power_well)
1056{
1057}
1058
1059static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
1060					   struct i915_power_well *power_well)
1061{
1062}
1063
1064static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
1065					     struct i915_power_well *power_well)
1066{
1067	return true;
1068}
1069
1070static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
1071					 struct i915_power_well *power_well)
1072{
1073	struct intel_display *display = &dev_priv->display;
1074
1075	if ((intel_de_read(display, TRANSCONF(dev_priv, PIPE_A)) & TRANSCONF_ENABLE) == 0)
1076		i830_enable_pipe(display, PIPE_A);
1077	if ((intel_de_read(display, TRANSCONF(dev_priv, PIPE_B)) & TRANSCONF_ENABLE) == 0)
1078		i830_enable_pipe(display, PIPE_B);
1079}
1080
1081static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
1082					  struct i915_power_well *power_well)
1083{
1084	struct intel_display *display = &dev_priv->display;
1085
1086	i830_disable_pipe(display, PIPE_B);
1087	i830_disable_pipe(display, PIPE_A);
1088}
1089
1090static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
1091					  struct i915_power_well *power_well)
1092{
1093	struct intel_display *display = &dev_priv->display;
1094
1095	return intel_de_read(display, TRANSCONF(dev_priv, PIPE_A)) & TRANSCONF_ENABLE &&
1096		intel_de_read(display, TRANSCONF(dev_priv, PIPE_B)) & TRANSCONF_ENABLE;
1097}
1098
1099static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
1100					  struct i915_power_well *power_well)
1101{
1102	if (intel_power_well_refcount(power_well) > 0)
1103		i830_pipes_power_well_enable(dev_priv, power_well);
1104	else
1105		i830_pipes_power_well_disable(dev_priv, power_well);
1106}
1107
1108static void vlv_set_power_well(struct drm_i915_private *dev_priv,
1109			       struct i915_power_well *power_well, bool enable)
1110{
1111	int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1112	u32 mask;
1113	u32 state;
1114	u32 ctrl;
1115
1116	mask = PUNIT_PWRGT_MASK(pw_idx);
1117	state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1118			 PUNIT_PWRGT_PWR_GATE(pw_idx);
1119
1120	vlv_punit_get(dev_priv);
1121
1122#define COND \
1123	((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1124
1125	if (COND)
1126		goto out;
1127
1128	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1129	ctrl &= ~mask;
1130	ctrl |= state;
1131	vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1132
1133	if (wait_for(COND, 100))
1134		drm_err(&dev_priv->drm,
1135			"timeout setting power well state %08x (%08x)\n",
1136			state,
1137			vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1138
1139#undef COND
1140
1141out:
1142	vlv_punit_put(dev_priv);
1143}
1144
1145static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1146				  struct i915_power_well *power_well)
1147{
1148	vlv_set_power_well(dev_priv, power_well, true);
1149}
1150
1151static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1152				   struct i915_power_well *power_well)
1153{
1154	vlv_set_power_well(dev_priv, power_well, false);
1155}
1156
1157static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1158				   struct i915_power_well *power_well)
1159{
1160	int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1161	bool enabled = false;
1162	u32 mask;
1163	u32 state;
1164	u32 ctrl;
1165
1166	mask = PUNIT_PWRGT_MASK(pw_idx);
1167	ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1168
1169	vlv_punit_get(dev_priv);
1170
1171	state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1172	/*
1173	 * We only ever set the power-on and power-gate states, anything
1174	 * else is unexpected.
1175	 */
1176	drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1177		    state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1178	if (state == ctrl)
1179		enabled = true;
1180
1181	/*
1182	 * A transient state at this point would mean some unexpected party
1183	 * is poking at the power controls too.
1184	 */
1185	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1186	drm_WARN_ON(&dev_priv->drm, ctrl != state);
1187
1188	vlv_punit_put(dev_priv);
1189
1190	return enabled;
1191}
1192
1193static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1194{
1195	/*
1196	 * On driver load, a pipe may be active and driving a DSI display.
1197	 * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1198	 * (and never recovering) in this case. intel_dsi_post_disable() will
1199	 * clear it when we turn off the display.
1200	 */
1201	intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
1202		     ~DPOUNIT_CLOCK_GATE_DISABLE, VRHUNIT_CLOCK_GATE_DISABLE);
1203
1204	/*
1205	 * Disable trickle feed and enable pnd deadline calculation
1206	 */
1207	intel_de_write(dev_priv, MI_ARB_VLV,
1208		       MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1209	intel_de_write(dev_priv, CBR1_VLV, 0);
1210
1211	drm_WARN_ON(&dev_priv->drm, DISPLAY_RUNTIME_INFO(dev_priv)->rawclk_freq == 0);
1212	intel_de_write(dev_priv, RAWCLK_FREQ_VLV,
1213		       DIV_ROUND_CLOSEST(DISPLAY_RUNTIME_INFO(dev_priv)->rawclk_freq,
1214					 1000));
1215}
1216
1217static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1218{
1219	struct intel_display *display = &dev_priv->display;
1220	struct intel_encoder *encoder;
1221	enum pipe pipe;
1222
1223	/*
1224	 * Enable the CRI clock source so we can get at the
1225	 * display and the reference clock for VGA
1226	 * hotplug / manual detection. Supposedly DSI also
1227	 * needs the ref clock up and running.
1228	 *
1229	 * CHV DPLL B/C have some issues if VGA mode is enabled.
1230	 */
1231	for_each_pipe(dev_priv, pipe) {
1232		u32 val = intel_de_read(dev_priv, DPLL(dev_priv, pipe));
1233
1234		val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1235		if (pipe != PIPE_A)
1236			val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1237
1238		intel_de_write(dev_priv, DPLL(dev_priv, pipe), val);
1239	}
1240
1241	vlv_init_display_clock_gating(dev_priv);
1242
1243	spin_lock_irq(&dev_priv->irq_lock);
1244	valleyview_enable_display_irqs(dev_priv);
1245	spin_unlock_irq(&dev_priv->irq_lock);
1246
1247	/*
1248	 * During driver initialization/resume we can avoid restoring the
1249	 * part of the HW/SW state that will be inited anyway explicitly.
1250	 */
1251	if (dev_priv->display.power.domains.initializing)
1252		return;
1253
1254	intel_hpd_init(dev_priv);
1255	intel_hpd_poll_disable(dev_priv);
1256
1257	/* Re-enable the ADPA, if we have one */
1258	for_each_intel_encoder(&dev_priv->drm, encoder) {
1259		if (encoder->type == INTEL_OUTPUT_ANALOG)
1260			intel_crt_reset(&encoder->base);
1261	}
1262
1263	intel_vga_redisable_power_on(display);
1264
1265	intel_pps_unlock_regs_wa(display);
1266}
1267
1268static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1269{
1270	struct intel_display *display = &dev_priv->display;
1271
1272	spin_lock_irq(&dev_priv->irq_lock);
1273	valleyview_disable_display_irqs(dev_priv);
1274	spin_unlock_irq(&dev_priv->irq_lock);
1275
1276	/* make sure we're done processing display irqs */
1277	intel_synchronize_irq(dev_priv);
1278
1279	vlv_pps_reset_all(display);
1280
1281	/* Prevent us from re-enabling polling on accident in late suspend */
1282	if (!dev_priv->drm.dev->power.is_suspended)
1283		intel_hpd_poll_enable(dev_priv);
1284}
1285
1286static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1287					  struct i915_power_well *power_well)
1288{
1289	vlv_set_power_well(dev_priv, power_well, true);
1290
1291	vlv_display_power_well_init(dev_priv);
1292}
1293
1294static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1295					   struct i915_power_well *power_well)
1296{
1297	vlv_display_power_well_deinit(dev_priv);
1298
1299	vlv_set_power_well(dev_priv, power_well, false);
1300}
1301
1302static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1303					   struct i915_power_well *power_well)
1304{
1305	/* since ref/cri clock was enabled */
1306	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1307
1308	vlv_set_power_well(dev_priv, power_well, true);
1309
1310	/*
1311	 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1312	 *  6.	De-assert cmn_reset/side_reset. Same as VLV X0.
1313	 *   a.	GUnit 0x2110 bit[0] set to 1 (def 0)
1314	 *   b.	The other bits such as sfr settings / modesel may all
1315	 *	be set to 0.
1316	 *
1317	 * This should only be done on init and resume from S3 with
1318	 * both PLLs disabled, or we risk losing DPIO and PLL
1319	 * synchronization.
1320	 */
1321	intel_de_rmw(dev_priv, DPIO_CTL, 0, DPIO_CMNRST);
1322}
1323
1324static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1325					    struct i915_power_well *power_well)
1326{
1327	enum pipe pipe;
1328
1329	for_each_pipe(dev_priv, pipe)
1330		assert_pll_disabled(dev_priv, pipe);
1331
1332	/* Assert common reset */
1333	intel_de_rmw(dev_priv, DPIO_CTL, DPIO_CMNRST, 0);
1334
1335	vlv_set_power_well(dev_priv, power_well, false);
1336}
1337
1338#define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1339
1340static void assert_chv_phy_status(struct intel_display *display)
1341{
1342	struct drm_i915_private *dev_priv = to_i915(display->drm);
1343	struct i915_power_well *cmn_bc =
1344		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1345	struct i915_power_well *cmn_d =
1346		lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1347	u32 phy_control = display->power.chv_phy_control;
1348	u32 phy_status = 0;
1349	u32 phy_status_mask = 0xffffffff;
1350
1351	/*
1352	 * The BIOS can leave the PHY is some weird state
1353	 * where it doesn't fully power down some parts.
1354	 * Disable the asserts until the PHY has been fully
1355	 * reset (ie. the power well has been disabled at
1356	 * least once).
1357	 */
1358	if (!display->power.chv_phy_assert[DPIO_PHY0])
1359		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1360				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1361				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1362				     PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1363				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1364				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1365
1366	if (!display->power.chv_phy_assert[DPIO_PHY1])
1367		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1368				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1369				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1370
1371	if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1372		phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1373
1374		/* this assumes override is only used to enable lanes */
1375		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1376			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1377
1378		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1379			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1380
1381		/* CL1 is on whenever anything is on in either channel */
1382		if (BITS_SET(phy_control,
1383			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1384			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1385			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1386
1387		/*
1388		 * The DPLLB check accounts for the pipe B + port A usage
1389		 * with CL2 powered up but all the lanes in the second channel
1390		 * powered down.
1391		 */
1392		if (BITS_SET(phy_control,
1393			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1394		    (intel_de_read(display, DPLL(display, PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1395			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1396
1397		if (BITS_SET(phy_control,
1398			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1399			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1400		if (BITS_SET(phy_control,
1401			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1402			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1403
1404		if (BITS_SET(phy_control,
1405			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1406			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1407		if (BITS_SET(phy_control,
1408			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1409			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1410	}
1411
1412	if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1413		phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1414
1415		/* this assumes override is only used to enable lanes */
1416		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1417			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1418
1419		if (BITS_SET(phy_control,
1420			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1421			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1422
1423		if (BITS_SET(phy_control,
1424			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1425			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1426		if (BITS_SET(phy_control,
1427			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1428			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1429	}
1430
1431	phy_status &= phy_status_mask;
1432
1433	/*
1434	 * The PHY may be busy with some initial calibration and whatnot,
1435	 * so the power state can take a while to actually change.
1436	 */
1437	if (intel_de_wait(display, DISPLAY_PHY_STATUS,
1438			  phy_status_mask, phy_status, 10))
1439		drm_err(display->drm,
1440			"Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1441			intel_de_read(display, DISPLAY_PHY_STATUS) & phy_status_mask,
1442			phy_status, display->power.chv_phy_control);
1443}
1444
1445#undef BITS_SET
1446
1447static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1448					   struct i915_power_well *power_well)
1449{
1450	struct intel_display *display = &dev_priv->display;
1451	enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1452	enum dpio_phy phy;
1453	u32 tmp;
1454
1455	drm_WARN_ON_ONCE(display->drm,
1456			 id != VLV_DISP_PW_DPIO_CMN_BC &&
1457			 id != CHV_DISP_PW_DPIO_CMN_D);
1458
1459	if (id == VLV_DISP_PW_DPIO_CMN_BC)
1460		phy = DPIO_PHY0;
1461	else
1462		phy = DPIO_PHY1;
1463
1464	/* since ref/cri clock was enabled */
1465	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1466	vlv_set_power_well(dev_priv, power_well, true);
1467
1468	/* Poll for phypwrgood signal */
1469	if (intel_de_wait_for_set(display, DISPLAY_PHY_STATUS,
1470				  PHY_POWERGOOD(phy), 1))
1471		drm_err(display->drm, "Display PHY %d is not power up\n",
1472			phy);
1473
1474	vlv_dpio_get(dev_priv);
1475
1476	/* Enable dynamic power down */
1477	tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW28);
1478	tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1479		DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1480	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW28, tmp);
1481
1482	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1483		tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW6_CH1);
1484		tmp |= DPIO_DYNPWRDOWNEN_CH1;
1485		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW6_CH1, tmp);
1486	} else {
1487		/*
1488		 * Force the non-existing CL2 off. BXT does this
1489		 * too, so maybe it saves some power even though
1490		 * CL2 doesn't exist?
1491		 */
1492		tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW30);
1493		tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1494		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW30, tmp);
1495	}
1496
1497	vlv_dpio_put(dev_priv);
1498
1499	display->power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1500	intel_de_write(display, DISPLAY_PHY_CONTROL,
1501		       display->power.chv_phy_control);
1502
1503	drm_dbg_kms(display->drm,
1504		    "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1505		    phy, display->power.chv_phy_control);
1506
1507	assert_chv_phy_status(display);
1508}
1509
1510static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1511					    struct i915_power_well *power_well)
1512{
1513	struct intel_display *display = &dev_priv->display;
1514	enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1515	enum dpio_phy phy;
1516
1517	drm_WARN_ON_ONCE(display->drm,
1518			 id != VLV_DISP_PW_DPIO_CMN_BC &&
1519			 id != CHV_DISP_PW_DPIO_CMN_D);
1520
1521	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1522		phy = DPIO_PHY0;
1523		assert_pll_disabled(dev_priv, PIPE_A);
1524		assert_pll_disabled(dev_priv, PIPE_B);
1525	} else {
1526		phy = DPIO_PHY1;
1527		assert_pll_disabled(dev_priv, PIPE_C);
1528	}
1529
1530	display->power.chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1531	intel_de_write(display, DISPLAY_PHY_CONTROL,
1532		       display->power.chv_phy_control);
1533
1534	vlv_set_power_well(dev_priv, power_well, false);
1535
1536	drm_dbg_kms(display->drm,
1537		    "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1538		    phy, display->power.chv_phy_control);
1539
1540	/* PHY is fully reset now, so we can enable the PHY state asserts */
1541	display->power.chv_phy_assert[phy] = true;
1542
1543	assert_chv_phy_status(display);
1544}
1545
1546static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1547				     enum dpio_channel ch, bool override, unsigned int mask)
1548{
1549	u32 reg, val, expected, actual;
1550
1551	/*
1552	 * The BIOS can leave the PHY is some weird state
1553	 * where it doesn't fully power down some parts.
1554	 * Disable the asserts until the PHY has been fully
1555	 * reset (ie. the power well has been disabled at
1556	 * least once).
1557	 */
1558	if (!dev_priv->display.power.chv_phy_assert[phy])
1559		return;
1560
1561	if (ch == DPIO_CH0)
1562		reg = CHV_CMN_DW0_CH0;
1563	else
1564		reg = CHV_CMN_DW6_CH1;
1565
1566	vlv_dpio_get(dev_priv);
1567	val = vlv_dpio_read(dev_priv, phy, reg);
1568	vlv_dpio_put(dev_priv);
1569
1570	/*
1571	 * This assumes !override is only used when the port is disabled.
1572	 * All lanes should power down even without the override when
1573	 * the port is disabled.
1574	 */
1575	if (!override || mask == 0xf) {
1576		expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1577		/*
1578		 * If CH1 common lane is not active anymore
1579		 * (eg. for pipe B DPLL) the entire channel will
1580		 * shut down, which causes the common lane registers
1581		 * to read as 0. That means we can't actually check
1582		 * the lane power down status bits, but as the entire
1583		 * register reads as 0 it's a good indication that the
1584		 * channel is indeed entirely powered down.
1585		 */
1586		if (ch == DPIO_CH1 && val == 0)
1587			expected = 0;
1588	} else if (mask != 0x0) {
1589		expected = DPIO_ANYDL_POWERDOWN;
1590	} else {
1591		expected = 0;
1592	}
1593
1594	if (ch == DPIO_CH0)
1595		actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH0 |
1596				       DPIO_ALLDL_POWERDOWN_CH0, val);
1597	else
1598		actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH1 |
1599				       DPIO_ALLDL_POWERDOWN_CH1, val);
1600
1601	drm_WARN(&dev_priv->drm, actual != expected,
1602		 "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1603		 !!(actual & DPIO_ALLDL_POWERDOWN),
1604		 !!(actual & DPIO_ANYDL_POWERDOWN),
1605		 !!(expected & DPIO_ALLDL_POWERDOWN),
1606		 !!(expected & DPIO_ANYDL_POWERDOWN),
1607		 reg, val);
1608}
1609
1610bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1611			  enum dpio_channel ch, bool override)
1612{
1613	struct intel_display *display = &dev_priv->display;
1614	struct i915_power_domains *power_domains = &display->power.domains;
1615	bool was_override;
1616
1617	mutex_lock(&power_domains->lock);
1618
1619	was_override = display->power.chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1620
1621	if (override == was_override)
1622		goto out;
1623
1624	if (override)
1625		display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1626	else
1627		display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1628
1629	intel_de_write(display, DISPLAY_PHY_CONTROL,
1630		       display->power.chv_phy_control);
1631
1632	drm_dbg_kms(display->drm,
1633		    "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1634		    phy, ch, display->power.chv_phy_control);
1635
1636	assert_chv_phy_status(display);
1637
1638out:
1639	mutex_unlock(&power_domains->lock);
1640
1641	return was_override;
1642}
1643
1644void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1645			     bool override, unsigned int mask)
1646{
1647	struct intel_display *display = to_intel_display(encoder);
1648	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1649	struct i915_power_domains *power_domains = &display->power.domains;
1650	enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
1651	enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
1652
1653	mutex_lock(&power_domains->lock);
1654
1655	display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1656	display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1657
1658	if (override)
1659		display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1660	else
1661		display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1662
1663	intel_de_write(display, DISPLAY_PHY_CONTROL,
1664		       display->power.chv_phy_control);
1665
1666	drm_dbg_kms(display->drm,
1667		    "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1668		    phy, ch, mask, display->power.chv_phy_control);
1669
1670	assert_chv_phy_status(display);
1671
1672	assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1673
1674	mutex_unlock(&power_domains->lock);
1675}
1676
1677static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1678					struct i915_power_well *power_well)
1679{
1680	enum pipe pipe = PIPE_A;
1681	bool enabled;
1682	u32 state, ctrl;
1683
1684	vlv_punit_get(dev_priv);
1685
1686	state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1687	/*
1688	 * We only ever set the power-on and power-gate states, anything
1689	 * else is unexpected.
1690	 */
1691	drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) &&
1692		    state != DP_SSS_PWR_GATE(pipe));
1693	enabled = state == DP_SSS_PWR_ON(pipe);
1694
1695	/*
1696	 * A transient state at this point would mean some unexpected party
1697	 * is poking at the power controls too.
1698	 */
1699	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1700	drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state);
1701
1702	vlv_punit_put(dev_priv);
1703
1704	return enabled;
1705}
1706
1707static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1708				    struct i915_power_well *power_well,
1709				    bool enable)
1710{
1711	enum pipe pipe = PIPE_A;
1712	u32 state;
1713	u32 ctrl;
1714
1715	state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1716
1717	vlv_punit_get(dev_priv);
1718
1719#define COND \
1720	((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1721
1722	if (COND)
1723		goto out;
1724
1725	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1726	ctrl &= ~DP_SSC_MASK(pipe);
1727	ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1728	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1729
1730	if (wait_for(COND, 100))
1731		drm_err(&dev_priv->drm,
1732			"timeout setting power well state %08x (%08x)\n",
1733			state,
1734			vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1735
1736#undef COND
1737
1738out:
1739	vlv_punit_put(dev_priv);
1740}
1741
1742static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1743					struct i915_power_well *power_well)
1744{
1745	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1746		       dev_priv->display.power.chv_phy_control);
1747}
1748
1749static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1750				       struct i915_power_well *power_well)
1751{
1752	chv_set_pipe_power_well(dev_priv, power_well, true);
1753
1754	vlv_display_power_well_init(dev_priv);
1755}
1756
1757static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1758					struct i915_power_well *power_well)
1759{
1760	vlv_display_power_well_deinit(dev_priv);
1761
1762	chv_set_pipe_power_well(dev_priv, power_well, false);
1763}
1764
1765static void
1766tgl_tc_cold_request(struct drm_i915_private *i915, bool block)
1767{
1768	u8 tries = 0;
1769	int ret;
1770
1771	while (1) {
1772		u32 low_val;
1773		u32 high_val = 0;
1774
1775		if (block)
1776			low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ;
1777		else
1778			low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ;
1779
1780		/*
1781		 * Spec states that we should timeout the request after 200us
1782		 * but the function below will timeout after 500us
1783		 */
1784		ret = snb_pcode_read(&i915->uncore, TGL_PCODE_TCCOLD, &low_val, &high_val);
1785		if (ret == 0) {
1786			if (block &&
1787			    (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
1788				ret = -EIO;
1789			else
1790				break;
1791		}
1792
1793		if (++tries == 3)
1794			break;
1795
1796		msleep(1);
1797	}
1798
1799	if (ret)
1800		drm_err(&i915->drm, "TC cold %sblock failed\n",
1801			block ? "" : "un");
1802	else
1803		drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n",
1804			    block ? "" : "un");
1805}
1806
1807static void
1808tgl_tc_cold_off_power_well_enable(struct drm_i915_private *i915,
1809				  struct i915_power_well *power_well)
1810{
1811	tgl_tc_cold_request(i915, true);
1812}
1813
1814static void
1815tgl_tc_cold_off_power_well_disable(struct drm_i915_private *i915,
1816				   struct i915_power_well *power_well)
1817{
1818	tgl_tc_cold_request(i915, false);
1819}
1820
1821static void
1822tgl_tc_cold_off_power_well_sync_hw(struct drm_i915_private *i915,
1823				   struct i915_power_well *power_well)
1824{
1825	if (intel_power_well_refcount(power_well) > 0)
1826		tgl_tc_cold_off_power_well_enable(i915, power_well);
1827	else
1828		tgl_tc_cold_off_power_well_disable(i915, power_well);
1829}
1830
1831static bool
1832tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv,
1833				      struct i915_power_well *power_well)
1834{
1835	/*
1836	 * Not the correctly implementation but there is no way to just read it
1837	 * from PCODE, so returning count to avoid state mismatch errors
1838	 */
1839	return intel_power_well_refcount(power_well);
1840}
1841
1842static void xelpdp_aux_power_well_enable(struct drm_i915_private *dev_priv,
1843					 struct i915_power_well *power_well)
1844{
1845	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1846	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
1847
1848	if (intel_phy_is_tc(dev_priv, phy))
1849		icl_tc_port_assert_ref_held(dev_priv, power_well,
1850					    aux_ch_to_digital_port(dev_priv, aux_ch));
1851
1852	intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch),
1853		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1854		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST);
1855
1856	/*
1857	 * The power status flag cannot be used to determine whether aux
1858	 * power wells have finished powering up.  Instead we're
1859	 * expected to just wait a fixed 600us after raising the request
1860	 * bit.
1861	 */
1862	usleep_range(600, 1200);
1863}
1864
1865static void xelpdp_aux_power_well_disable(struct drm_i915_private *dev_priv,
1866					  struct i915_power_well *power_well)
1867{
1868	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1869
1870	intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch),
1871		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1872		     0);
1873	usleep_range(10, 30);
1874}
1875
1876static bool xelpdp_aux_power_well_enabled(struct drm_i915_private *dev_priv,
1877					  struct i915_power_well *power_well)
1878{
1879	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1880
1881	return intel_de_read(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch)) &
1882		XELPDP_DP_AUX_CH_CTL_POWER_STATUS;
1883}
1884
1885static void xe2lpd_pica_power_well_enable(struct drm_i915_private *dev_priv,
1886					  struct i915_power_well *power_well)
1887{
1888	intel_de_write(dev_priv, XE2LPD_PICA_PW_CTL,
1889		       XE2LPD_PICA_CTL_POWER_REQUEST);
1890
1891	if (intel_de_wait_for_set(dev_priv, XE2LPD_PICA_PW_CTL,
1892				  XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1893		drm_dbg_kms(&dev_priv->drm, "pica power well enable timeout\n");
1894
1895		drm_WARN(&dev_priv->drm, 1, "Power well PICA timeout when enabled");
1896	}
1897}
1898
1899static void xe2lpd_pica_power_well_disable(struct drm_i915_private *dev_priv,
1900					   struct i915_power_well *power_well)
1901{
1902	intel_de_write(dev_priv, XE2LPD_PICA_PW_CTL, 0);
1903
1904	if (intel_de_wait_for_clear(dev_priv, XE2LPD_PICA_PW_CTL,
1905				    XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1906		drm_dbg_kms(&dev_priv->drm, "pica power well disable timeout\n");
1907
1908		drm_WARN(&dev_priv->drm, 1, "Power well PICA timeout when disabled");
1909	}
1910}
1911
1912static bool xe2lpd_pica_power_well_enabled(struct drm_i915_private *dev_priv,
1913					   struct i915_power_well *power_well)
1914{
1915	return intel_de_read(dev_priv, XE2LPD_PICA_PW_CTL) &
1916		XE2LPD_PICA_CTL_POWER_STATUS;
1917}
1918
1919const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1920	.sync_hw = i9xx_power_well_sync_hw_noop,
1921	.enable = i9xx_always_on_power_well_noop,
1922	.disable = i9xx_always_on_power_well_noop,
1923	.is_enabled = i9xx_always_on_power_well_enabled,
1924};
1925
1926const struct i915_power_well_ops chv_pipe_power_well_ops = {
1927	.sync_hw = chv_pipe_power_well_sync_hw,
1928	.enable = chv_pipe_power_well_enable,
1929	.disable = chv_pipe_power_well_disable,
1930	.is_enabled = chv_pipe_power_well_enabled,
1931};
1932
1933const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1934	.sync_hw = i9xx_power_well_sync_hw_noop,
1935	.enable = chv_dpio_cmn_power_well_enable,
1936	.disable = chv_dpio_cmn_power_well_disable,
1937	.is_enabled = vlv_power_well_enabled,
1938};
1939
1940const struct i915_power_well_ops i830_pipes_power_well_ops = {
1941	.sync_hw = i830_pipes_power_well_sync_hw,
1942	.enable = i830_pipes_power_well_enable,
1943	.disable = i830_pipes_power_well_disable,
1944	.is_enabled = i830_pipes_power_well_enabled,
1945};
1946
1947static const struct i915_power_well_regs hsw_power_well_regs = {
1948	.bios	= HSW_PWR_WELL_CTL1,
1949	.driver	= HSW_PWR_WELL_CTL2,
1950	.kvmr	= HSW_PWR_WELL_CTL3,
1951	.debug	= HSW_PWR_WELL_CTL4,
1952};
1953
1954const struct i915_power_well_ops hsw_power_well_ops = {
1955	.regs = &hsw_power_well_regs,
1956	.sync_hw = hsw_power_well_sync_hw,
1957	.enable = hsw_power_well_enable,
1958	.disable = hsw_power_well_disable,
1959	.is_enabled = hsw_power_well_enabled,
1960};
1961
1962const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1963	.sync_hw = i9xx_power_well_sync_hw_noop,
1964	.enable = gen9_dc_off_power_well_enable,
1965	.disable = gen9_dc_off_power_well_disable,
1966	.is_enabled = gen9_dc_off_power_well_enabled,
1967};
1968
1969const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1970	.sync_hw = i9xx_power_well_sync_hw_noop,
1971	.enable = bxt_dpio_cmn_power_well_enable,
1972	.disable = bxt_dpio_cmn_power_well_disable,
1973	.is_enabled = bxt_dpio_cmn_power_well_enabled,
1974};
1975
1976const struct i915_power_well_ops vlv_display_power_well_ops = {
1977	.sync_hw = i9xx_power_well_sync_hw_noop,
1978	.enable = vlv_display_power_well_enable,
1979	.disable = vlv_display_power_well_disable,
1980	.is_enabled = vlv_power_well_enabled,
1981};
1982
1983const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1984	.sync_hw = i9xx_power_well_sync_hw_noop,
1985	.enable = vlv_dpio_cmn_power_well_enable,
1986	.disable = vlv_dpio_cmn_power_well_disable,
1987	.is_enabled = vlv_power_well_enabled,
1988};
1989
1990const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1991	.sync_hw = i9xx_power_well_sync_hw_noop,
1992	.enable = vlv_power_well_enable,
1993	.disable = vlv_power_well_disable,
1994	.is_enabled = vlv_power_well_enabled,
1995};
1996
1997static const struct i915_power_well_regs icl_aux_power_well_regs = {
1998	.bios	= ICL_PWR_WELL_CTL_AUX1,
1999	.driver	= ICL_PWR_WELL_CTL_AUX2,
2000	.debug	= ICL_PWR_WELL_CTL_AUX4,
2001};
2002
2003const struct i915_power_well_ops icl_aux_power_well_ops = {
2004	.regs = &icl_aux_power_well_regs,
2005	.sync_hw = hsw_power_well_sync_hw,
2006	.enable = icl_aux_power_well_enable,
2007	.disable = icl_aux_power_well_disable,
2008	.is_enabled = hsw_power_well_enabled,
2009};
2010
2011static const struct i915_power_well_regs icl_ddi_power_well_regs = {
2012	.bios	= ICL_PWR_WELL_CTL_DDI1,
2013	.driver	= ICL_PWR_WELL_CTL_DDI2,
2014	.debug	= ICL_PWR_WELL_CTL_DDI4,
2015};
2016
2017const struct i915_power_well_ops icl_ddi_power_well_ops = {
2018	.regs = &icl_ddi_power_well_regs,
2019	.sync_hw = hsw_power_well_sync_hw,
2020	.enable = hsw_power_well_enable,
2021	.disable = hsw_power_well_disable,
2022	.is_enabled = hsw_power_well_enabled,
2023};
2024
2025const struct i915_power_well_ops tgl_tc_cold_off_ops = {
2026	.sync_hw = tgl_tc_cold_off_power_well_sync_hw,
2027	.enable = tgl_tc_cold_off_power_well_enable,
2028	.disable = tgl_tc_cold_off_power_well_disable,
2029	.is_enabled = tgl_tc_cold_off_power_well_is_enabled,
2030};
2031
2032const struct i915_power_well_ops xelpdp_aux_power_well_ops = {
2033	.sync_hw = i9xx_power_well_sync_hw_noop,
2034	.enable = xelpdp_aux_power_well_enable,
2035	.disable = xelpdp_aux_power_well_disable,
2036	.is_enabled = xelpdp_aux_power_well_enabled,
2037};
2038
2039const struct i915_power_well_ops xe2lpd_pica_power_well_ops = {
2040	.sync_hw = i9xx_power_well_sync_hw_noop,
2041	.enable = xe2lpd_pica_power_well_enable,
2042	.disable = xe2lpd_pica_power_well_disable,
2043	.is_enabled = xe2lpd_pica_power_well_enabled,
2044};