1/*
   2 * Copyright © 2006-2017 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21 * DEALINGS IN THE SOFTWARE.
  22 */
  23
  24#include <linux/time.h>
  25
  26#include "intel_atomic.h"
  27#include "intel_bw.h"
  28#include "intel_cdclk.h"
  29#include "intel_de.h"
  30#include "intel_display_types.h"
  31#include "intel_psr.h"
  32#include "intel_sideband.h"
  33
  34/**
  35 * DOC: CDCLK / RAWCLK
  36 *
  37 * The display engine uses several different clocks to do its work. There
  38 * are two main clocks involved that aren't directly related to the actual
  39 * pixel clock or any symbol/bit clock of the actual output port. These
  40 * are the core display clock (CDCLK) and RAWCLK.
  41 *
  42 * CDCLK clocks most of the display pipe logic, and thus its frequency
  43 * must be high enough to support the rate at which pixels are flowing
  44 * through the pipes. Downscaling must also be accounted as that increases
  45 * the effective pixel rate.
  46 *
  47 * On several platforms the CDCLK frequency can be changed dynamically
  48 * to minimize power consumption for a given display configuration.
  49 * Typically changes to the CDCLK frequency require all the display pipes
  50 * to be shut down while the frequency is being changed.
  51 *
  52 * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  53 * DMC will not change the active CDCLK frequency however, so that part
  54 * will still be performed by the driver directly.
  55 *
  56 * RAWCLK is a fixed frequency clock, often used by various auxiliary
  57 * blocks such as AUX CH or backlight PWM. Hence the only thing we
  58 * really need to know about RAWCLK is its frequency so that various
  59 * dividers can be programmed correctly.
  60 */
  61
  62static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
  63				   struct intel_cdclk_config *cdclk_config)
  64{
  65	cdclk_config->cdclk = 133333;
  66}
  67
  68static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
  69				   struct intel_cdclk_config *cdclk_config)
  70{
  71	cdclk_config->cdclk = 200000;
  72}
  73
  74static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
  75				   struct intel_cdclk_config *cdclk_config)
  76{
  77	cdclk_config->cdclk = 266667;
  78}
  79
  80static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
  81				   struct intel_cdclk_config *cdclk_config)
  82{
  83	cdclk_config->cdclk = 333333;
  84}
  85
  86static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
  87				   struct intel_cdclk_config *cdclk_config)
  88{
  89	cdclk_config->cdclk = 400000;
  90}
  91
  92static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
  93				   struct intel_cdclk_config *cdclk_config)
  94{
  95	cdclk_config->cdclk = 450000;
  96}
  97
  98static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
  99			   struct intel_cdclk_config *cdclk_config)
 100{
 101	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 102	u16 hpllcc = 0;
 103
 104	/*
 105	 * 852GM/852GMV only supports 133 MHz and the HPLLCC
 106	 * encoding is different :(
 107	 * FIXME is this the right way to detect 852GM/852GMV?
 108	 */
 109	if (pdev->revision == 0x1) {
 110		cdclk_config->cdclk = 133333;
 111		return;
 112	}
 113
 114	pci_bus_read_config_word(pdev->bus,
 115				 PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 116
 117	/* Assume that the hardware is in the high speed state.  This
 118	 * should be the default.
 119	 */
 120	switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 121	case GC_CLOCK_133_200:
 122	case GC_CLOCK_133_200_2:
 123	case GC_CLOCK_100_200:
 124		cdclk_config->cdclk = 200000;
 125		break;
 126	case GC_CLOCK_166_250:
 127		cdclk_config->cdclk = 250000;
 128		break;
 129	case GC_CLOCK_100_133:
 130		cdclk_config->cdclk = 133333;
 131		break;
 132	case GC_CLOCK_133_266:
 133	case GC_CLOCK_133_266_2:
 134	case GC_CLOCK_166_266:
 135		cdclk_config->cdclk = 266667;
 136		break;
 137	}
 138}
 139
 140static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 141			     struct intel_cdclk_config *cdclk_config)
 142{
 143	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 144	u16 gcfgc = 0;
 145
 146	pci_read_config_word(pdev, GCFGC, &gcfgc);
 147
 148	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 149		cdclk_config->cdclk = 133333;
 150		return;
 151	}
 152
 153	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 154	case GC_DISPLAY_CLOCK_333_320_MHZ:
 155		cdclk_config->cdclk = 333333;
 156		break;
 157	default:
 158	case GC_DISPLAY_CLOCK_190_200_MHZ:
 159		cdclk_config->cdclk = 190000;
 160		break;
 161	}
 162}
 163
 164static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 165			     struct intel_cdclk_config *cdclk_config)
 166{
 167	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 168	u16 gcfgc = 0;
 169
 170	pci_read_config_word(pdev, GCFGC, &gcfgc);
 171
 172	if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 173		cdclk_config->cdclk = 133333;
 174		return;
 175	}
 176
 177	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 178	case GC_DISPLAY_CLOCK_333_320_MHZ:
 179		cdclk_config->cdclk = 320000;
 180		break;
 181	default:
 182	case GC_DISPLAY_CLOCK_190_200_MHZ:
 183		cdclk_config->cdclk = 200000;
 184		break;
 185	}
 186}
 187
 188static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 189{
 190	static const unsigned int blb_vco[8] = {
 191		[0] = 3200000,
 192		[1] = 4000000,
 193		[2] = 5333333,
 194		[3] = 4800000,
 195		[4] = 6400000,
 196	};
 197	static const unsigned int pnv_vco[8] = {
 198		[0] = 3200000,
 199		[1] = 4000000,
 200		[2] = 5333333,
 201		[3] = 4800000,
 202		[4] = 2666667,
 203	};
 204	static const unsigned int cl_vco[8] = {
 205		[0] = 3200000,
 206		[1] = 4000000,
 207		[2] = 5333333,
 208		[3] = 6400000,
 209		[4] = 3333333,
 210		[5] = 3566667,
 211		[6] = 4266667,
 212	};
 213	static const unsigned int elk_vco[8] = {
 214		[0] = 3200000,
 215		[1] = 4000000,
 216		[2] = 5333333,
 217		[3] = 4800000,
 218	};
 219	static const unsigned int ctg_vco[8] = {
 220		[0] = 3200000,
 221		[1] = 4000000,
 222		[2] = 5333333,
 223		[3] = 6400000,
 224		[4] = 2666667,
 225		[5] = 4266667,
 226	};
 227	const unsigned int *vco_table;
 228	unsigned int vco;
 229	u8 tmp = 0;
 230
 231	/* FIXME other chipsets? */
 232	if (IS_GM45(dev_priv))
 233		vco_table = ctg_vco;
 234	else if (IS_G45(dev_priv))
 235		vco_table = elk_vco;
 236	else if (IS_I965GM(dev_priv))
 237		vco_table = cl_vco;
 238	else if (IS_PINEVIEW(dev_priv))
 239		vco_table = pnv_vco;
 240	else if (IS_G33(dev_priv))
 241		vco_table = blb_vco;
 242	else
 243		return 0;
 244
 245	tmp = intel_de_read(dev_priv,
 246			    IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 247
 248	vco = vco_table[tmp & 0x7];
 249	if (vco == 0)
 250		drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 251			tmp);
 252	else
 253		drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 254
 255	return vco;
 256}
 257
 258static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 259			  struct intel_cdclk_config *cdclk_config)
 260{
 261	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 262	static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 263	static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 264	static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 265	static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 266	const u8 *div_table;
 267	unsigned int cdclk_sel;
 268	u16 tmp = 0;
 269
 270	cdclk_config->vco = intel_hpll_vco(dev_priv);
 271
 272	pci_read_config_word(pdev, GCFGC, &tmp);
 273
 274	cdclk_sel = (tmp >> 4) & 0x7;
 275
 276	if (cdclk_sel >= ARRAY_SIZE(div_3200))
 277		goto fail;
 278
 279	switch (cdclk_config->vco) {
 280	case 3200000:
 281		div_table = div_3200;
 282		break;
 283	case 4000000:
 284		div_table = div_4000;
 285		break;
 286	case 4800000:
 287		div_table = div_4800;
 288		break;
 289	case 5333333:
 290		div_table = div_5333;
 291		break;
 292	default:
 293		goto fail;
 294	}
 295
 296	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 297						div_table[cdclk_sel]);
 298	return;
 299
 300fail:
 301	drm_err(&dev_priv->drm,
 302		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 303		cdclk_config->vco, tmp);
 304	cdclk_config->cdclk = 190476;
 305}
 306
 307static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 308			  struct intel_cdclk_config *cdclk_config)
 309{
 310	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 311	u16 gcfgc = 0;
 312
 313	pci_read_config_word(pdev, GCFGC, &gcfgc);
 314
 315	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 316	case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 317		cdclk_config->cdclk = 266667;
 318		break;
 319	case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 320		cdclk_config->cdclk = 333333;
 321		break;
 322	case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 323		cdclk_config->cdclk = 444444;
 324		break;
 325	case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 326		cdclk_config->cdclk = 200000;
 327		break;
 328	default:
 329		drm_err(&dev_priv->drm,
 330			"Unknown pnv display core clock 0x%04x\n", gcfgc);
 331		fallthrough;
 332	case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 333		cdclk_config->cdclk = 133333;
 334		break;
 335	case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 336		cdclk_config->cdclk = 166667;
 337		break;
 338	}
 339}
 340
 341static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 342			     struct intel_cdclk_config *cdclk_config)
 343{
 344	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 345	static const u8 div_3200[] = { 16, 10,  8 };
 346	static const u8 div_4000[] = { 20, 12, 10 };
 347	static const u8 div_5333[] = { 24, 16, 14 };
 348	const u8 *div_table;
 349	unsigned int cdclk_sel;
 350	u16 tmp = 0;
 351
 352	cdclk_config->vco = intel_hpll_vco(dev_priv);
 353
 354	pci_read_config_word(pdev, GCFGC, &tmp);
 355
 356	cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 357
 358	if (cdclk_sel >= ARRAY_SIZE(div_3200))
 359		goto fail;
 360
 361	switch (cdclk_config->vco) {
 362	case 3200000:
 363		div_table = div_3200;
 364		break;
 365	case 4000000:
 366		div_table = div_4000;
 367		break;
 368	case 5333333:
 369		div_table = div_5333;
 370		break;
 371	default:
 372		goto fail;
 373	}
 374
 375	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 376						div_table[cdclk_sel]);
 377	return;
 378
 379fail:
 380	drm_err(&dev_priv->drm,
 381		"Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 382		cdclk_config->vco, tmp);
 383	cdclk_config->cdclk = 200000;
 384}
 385
 386static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 387			   struct intel_cdclk_config *cdclk_config)
 388{
 389	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 390	unsigned int cdclk_sel;
 391	u16 tmp = 0;
 392
 393	cdclk_config->vco = intel_hpll_vco(dev_priv);
 394
 395	pci_read_config_word(pdev, GCFGC, &tmp);
 396
 397	cdclk_sel = (tmp >> 12) & 0x1;
 398
 399	switch (cdclk_config->vco) {
 400	case 2666667:
 401	case 4000000:
 402	case 5333333:
 403		cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 404		break;
 405	case 3200000:
 406		cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 407		break;
 408	default:
 409		drm_err(&dev_priv->drm,
 410			"Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 411			cdclk_config->vco, tmp);
 412		cdclk_config->cdclk = 222222;
 413		break;
 414	}
 415}
 416
 417static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 418			  struct intel_cdclk_config *cdclk_config)
 419{
 420	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 421	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 422
 423	if (lcpll & LCPLL_CD_SOURCE_FCLK)
 424		cdclk_config->cdclk = 800000;
 425	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 426		cdclk_config->cdclk = 450000;
 427	else if (freq == LCPLL_CLK_FREQ_450)
 428		cdclk_config->cdclk = 450000;
 429	else if (IS_HSW_ULT(dev_priv))
 430		cdclk_config->cdclk = 337500;
 431	else
 432		cdclk_config->cdclk = 540000;
 433}
 434
 435static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 436{
 437	int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 438		333333 : 320000;
 439
 440	/*
 441	 * We seem to get an unstable or solid color picture at 200MHz.
 442	 * Not sure what's wrong. For now use 200MHz only when all pipes
 443	 * are off.
 444	 */
 445	if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 446		return 400000;
 447	else if (min_cdclk > 266667)
 448		return freq_320;
 449	else if (min_cdclk > 0)
 450		return 266667;
 451	else
 452		return 200000;
 453}
 454
 455static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 456{
 457	if (IS_VALLEYVIEW(dev_priv)) {
 458		if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 459			return 2;
 460		else if (cdclk >= 266667)
 461			return 1;
 462		else
 463			return 0;
 464	} else {
 465		/*
 466		 * Specs are full of misinformation, but testing on actual
 467		 * hardware has shown that we just need to write the desired
 468		 * CCK divider into the Punit register.
 469		 */
 470		return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 471	}
 472}
 473
 474static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 475			  struct intel_cdclk_config *cdclk_config)
 476{
 477	u32 val;
 478
 479	vlv_iosf_sb_get(dev_priv,
 480			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 481
 482	cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 483	cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 484						CCK_DISPLAY_CLOCK_CONTROL,
 485						cdclk_config->vco);
 486
 487	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 488
 489	vlv_iosf_sb_put(dev_priv,
 490			BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 491
 492	if (IS_VALLEYVIEW(dev_priv))
 493		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 494			DSPFREQGUAR_SHIFT;
 495	else
 496		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 497			DSPFREQGUAR_SHIFT_CHV;
 498}
 499
 500static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 501{
 502	unsigned int credits, default_credits;
 503
 504	if (IS_CHERRYVIEW(dev_priv))
 505		default_credits = PFI_CREDIT(12);
 506	else
 507		default_credits = PFI_CREDIT(8);
 508
 509	if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 510		/* CHV suggested value is 31 or 63 */
 511		if (IS_CHERRYVIEW(dev_priv))
 512			credits = PFI_CREDIT_63;
 513		else
 514			credits = PFI_CREDIT(15);
 515	} else {
 516		credits = default_credits;
 517	}
 518
 519	/*
 520	 * WA - write default credits before re-programming
 521	 * FIXME: should we also set the resend bit here?
 522	 */
 523	intel_de_write(dev_priv, GCI_CONTROL,
 524		       VGA_FAST_MODE_DISABLE | default_credits);
 525
 526	intel_de_write(dev_priv, GCI_CONTROL,
 527		       VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 528
 529	/*
 530	 * FIXME is this guaranteed to clear
 531	 * immediately or should we poll for it?
 532	 */
 533	drm_WARN_ON(&dev_priv->drm,
 534		    intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 535}
 536
 537static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 538			  const struct intel_cdclk_config *cdclk_config,
 539			  enum pipe pipe)
 540{
 541	int cdclk = cdclk_config->cdclk;
 542	u32 val, cmd = cdclk_config->voltage_level;
 543	intel_wakeref_t wakeref;
 544
 545	switch (cdclk) {
 546	case 400000:
 547	case 333333:
 548	case 320000:
 549	case 266667:
 550	case 200000:
 551		break;
 552	default:
 553		MISSING_CASE(cdclk);
 554		return;
 555	}
 556
 557	/* There are cases where we can end up here with power domains
 558	 * off and a CDCLK frequency other than the minimum, like when
 559	 * issuing a modeset without actually changing any display after
 560	 * a system suspend.  So grab the display core domain, which covers
 561	 * the HW blocks needed for the following programming.
 562	 */
 563	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 564
 565	vlv_iosf_sb_get(dev_priv,
 566			BIT(VLV_IOSF_SB_CCK) |
 567			BIT(VLV_IOSF_SB_BUNIT) |
 568			BIT(VLV_IOSF_SB_PUNIT));
 569
 570	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 571	val &= ~DSPFREQGUAR_MASK;
 572	val |= (cmd << DSPFREQGUAR_SHIFT);
 573	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 574	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 575		      DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 576		     50)) {
 577		drm_err(&dev_priv->drm,
 578			"timed out waiting for CDclk change\n");
 579	}
 580
 581	if (cdclk == 400000) {
 582		u32 divider;
 583
 584		divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 585					    cdclk) - 1;
 586
 587		/* adjust cdclk divider */
 588		val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 589		val &= ~CCK_FREQUENCY_VALUES;
 590		val |= divider;
 591		vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 592
 593		if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 594			      CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 595			     50))
 596			drm_err(&dev_priv->drm,
 597				"timed out waiting for CDclk change\n");
 598	}
 599
 600	/* adjust self-refresh exit latency value */
 601	val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 602	val &= ~0x7f;
 603
 604	/*
 605	 * For high bandwidth configs, we set a higher latency in the bunit
 606	 * so that the core display fetch happens in time to avoid underruns.
 607	 */
 608	if (cdclk == 400000)
 609		val |= 4500 / 250; /* 4.5 usec */
 610	else
 611		val |= 3000 / 250; /* 3.0 usec */
 612	vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 613
 614	vlv_iosf_sb_put(dev_priv,
 615			BIT(VLV_IOSF_SB_CCK) |
 616			BIT(VLV_IOSF_SB_BUNIT) |
 617			BIT(VLV_IOSF_SB_PUNIT));
 618
 619	intel_update_cdclk(dev_priv);
 620
 621	vlv_program_pfi_credits(dev_priv);
 622
 623	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 624}
 625
 626static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 627			  const struct intel_cdclk_config *cdclk_config,
 628			  enum pipe pipe)
 629{
 630	int cdclk = cdclk_config->cdclk;
 631	u32 val, cmd = cdclk_config->voltage_level;
 632	intel_wakeref_t wakeref;
 633
 634	switch (cdclk) {
 635	case 333333:
 636	case 320000:
 637	case 266667:
 638	case 200000:
 639		break;
 640	default:
 641		MISSING_CASE(cdclk);
 642		return;
 643	}
 644
 645	/* There are cases where we can end up here with power domains
 646	 * off and a CDCLK frequency other than the minimum, like when
 647	 * issuing a modeset without actually changing any display after
 648	 * a system suspend.  So grab the display core domain, which covers
 649	 * the HW blocks needed for the following programming.
 650	 */
 651	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 652
 653	vlv_punit_get(dev_priv);
 654	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 655	val &= ~DSPFREQGUAR_MASK_CHV;
 656	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 657	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 658	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 659		      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 660		     50)) {
 661		drm_err(&dev_priv->drm,
 662			"timed out waiting for CDclk change\n");
 663	}
 664
 665	vlv_punit_put(dev_priv);
 666
 667	intel_update_cdclk(dev_priv);
 668
 669	vlv_program_pfi_credits(dev_priv);
 670
 671	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 672}
 673
 674static int bdw_calc_cdclk(int min_cdclk)
 675{
 676	if (min_cdclk > 540000)
 677		return 675000;
 678	else if (min_cdclk > 450000)
 679		return 540000;
 680	else if (min_cdclk > 337500)
 681		return 450000;
 682	else
 683		return 337500;
 684}
 685
 686static u8 bdw_calc_voltage_level(int cdclk)
 687{
 688	switch (cdclk) {
 689	default:
 690	case 337500:
 691		return 2;
 692	case 450000:
 693		return 0;
 694	case 540000:
 695		return 1;
 696	case 675000:
 697		return 3;
 698	}
 699}
 700
 701static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 702			  struct intel_cdclk_config *cdclk_config)
 703{
 704	u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 705	u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 706
 707	if (lcpll & LCPLL_CD_SOURCE_FCLK)
 708		cdclk_config->cdclk = 800000;
 709	else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 710		cdclk_config->cdclk = 450000;
 711	else if (freq == LCPLL_CLK_FREQ_450)
 712		cdclk_config->cdclk = 450000;
 713	else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 714		cdclk_config->cdclk = 540000;
 715	else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 716		cdclk_config->cdclk = 337500;
 717	else
 718		cdclk_config->cdclk = 675000;
 719
 720	/*
 721	 * Can't read this out :( Let's assume it's
 722	 * at least what the CDCLK frequency requires.
 723	 */
 724	cdclk_config->voltage_level =
 725		bdw_calc_voltage_level(cdclk_config->cdclk);
 726}
 727
 728static u32 bdw_cdclk_freq_sel(int cdclk)
 729{
 730	switch (cdclk) {
 731	default:
 732		MISSING_CASE(cdclk);
 733		fallthrough;
 734	case 337500:
 735		return LCPLL_CLK_FREQ_337_5_BDW;
 736	case 450000:
 737		return LCPLL_CLK_FREQ_450;
 738	case 540000:
 739		return LCPLL_CLK_FREQ_54O_BDW;
 740	case 675000:
 741		return LCPLL_CLK_FREQ_675_BDW;
 742	}
 743}
 744
 745static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 746			  const struct intel_cdclk_config *cdclk_config,
 747			  enum pipe pipe)
 748{
 749	int cdclk = cdclk_config->cdclk;
 750	int ret;
 751
 752	if (drm_WARN(&dev_priv->drm,
 753		     (intel_de_read(dev_priv, LCPLL_CTL) &
 754		      (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 755		       LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 756		       LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 757		       LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 758		     "trying to change cdclk frequency with cdclk not enabled\n"))
 759		return;
 760
 761	ret = sandybridge_pcode_write(dev_priv,
 762				      BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 763	if (ret) {
 764		drm_err(&dev_priv->drm,
 765			"failed to inform pcode about cdclk change\n");
 766		return;
 767	}
 768
 769	intel_de_rmw(dev_priv, LCPLL_CTL,
 770		     0, LCPLL_CD_SOURCE_FCLK);
 771
 772	/*
 773	 * According to the spec, it should be enough to poll for this 1 us.
 774	 * However, extensive testing shows that this can take longer.
 775	 */
 776	if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 777			LCPLL_CD_SOURCE_FCLK_DONE, 100))
 778		drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 779
 780	intel_de_rmw(dev_priv, LCPLL_CTL,
 781		     LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 782
 783	intel_de_rmw(dev_priv, LCPLL_CTL,
 784		     LCPLL_CD_SOURCE_FCLK, 0);
 785
 786	if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 787			 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 788		drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 789
 790	sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
 791				cdclk_config->voltage_level);
 792
 793	intel_de_write(dev_priv, CDCLK_FREQ,
 794		       DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 795
 796	intel_update_cdclk(dev_priv);
 797}
 798
 799static int skl_calc_cdclk(int min_cdclk, int vco)
 800{
 801	if (vco == 8640000) {
 802		if (min_cdclk > 540000)
 803			return 617143;
 804		else if (min_cdclk > 432000)
 805			return 540000;
 806		else if (min_cdclk > 308571)
 807			return 432000;
 808		else
 809			return 308571;
 810	} else {
 811		if (min_cdclk > 540000)
 812			return 675000;
 813		else if (min_cdclk > 450000)
 814			return 540000;
 815		else if (min_cdclk > 337500)
 816			return 450000;
 817		else
 818			return 337500;
 819	}
 820}
 821
 822static u8 skl_calc_voltage_level(int cdclk)
 823{
 824	if (cdclk > 540000)
 825		return 3;
 826	else if (cdclk > 450000)
 827		return 2;
 828	else if (cdclk > 337500)
 829		return 1;
 830	else
 831		return 0;
 832}
 833
 834static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 835			     struct intel_cdclk_config *cdclk_config)
 836{
 837	u32 val;
 838
 839	cdclk_config->ref = 24000;
 840	cdclk_config->vco = 0;
 841
 842	val = intel_de_read(dev_priv, LCPLL1_CTL);
 843	if ((val & LCPLL_PLL_ENABLE) == 0)
 844		return;
 845
 846	if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 847		return;
 848
 849	val = intel_de_read(dev_priv, DPLL_CTRL1);
 850
 851	if (drm_WARN_ON(&dev_priv->drm,
 852			(val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 853				DPLL_CTRL1_SSC(SKL_DPLL0) |
 854				DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 855			DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 856		return;
 857
 858	switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 859	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 860	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 861	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 862	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 863		cdclk_config->vco = 8100000;
 864		break;
 865	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 866	case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 867		cdclk_config->vco = 8640000;
 868		break;
 869	default:
 870		MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 871		break;
 872	}
 873}
 874
 875static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 876			  struct intel_cdclk_config *cdclk_config)
 877{
 878	u32 cdctl;
 879
 880	skl_dpll0_update(dev_priv, cdclk_config);
 881
 882	cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 883
 884	if (cdclk_config->vco == 0)
 885		goto out;
 886
 887	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 888
 889	if (cdclk_config->vco == 8640000) {
 890		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 891		case CDCLK_FREQ_450_432:
 892			cdclk_config->cdclk = 432000;
 893			break;
 894		case CDCLK_FREQ_337_308:
 895			cdclk_config->cdclk = 308571;
 896			break;
 897		case CDCLK_FREQ_540:
 898			cdclk_config->cdclk = 540000;
 899			break;
 900		case CDCLK_FREQ_675_617:
 901			cdclk_config->cdclk = 617143;
 902			break;
 903		default:
 904			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 905			break;
 906		}
 907	} else {
 908		switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 909		case CDCLK_FREQ_450_432:
 910			cdclk_config->cdclk = 450000;
 911			break;
 912		case CDCLK_FREQ_337_308:
 913			cdclk_config->cdclk = 337500;
 914			break;
 915		case CDCLK_FREQ_540:
 916			cdclk_config->cdclk = 540000;
 917			break;
 918		case CDCLK_FREQ_675_617:
 919			cdclk_config->cdclk = 675000;
 920			break;
 921		default:
 922			MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 923			break;
 924		}
 925	}
 926
 927 out:
 928	/*
 929	 * Can't read this out :( Let's assume it's
 930	 * at least what the CDCLK frequency requires.
 931	 */
 932	cdclk_config->voltage_level =
 933		skl_calc_voltage_level(cdclk_config->cdclk);
 934}
 935
 936/* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 937static int skl_cdclk_decimal(int cdclk)
 938{
 939	return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 940}
 941
 942static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 943					int vco)
 944{
 945	bool changed = dev_priv->skl_preferred_vco_freq != vco;
 946
 947	dev_priv->skl_preferred_vco_freq = vco;
 948
 949	if (changed)
 950		intel_update_max_cdclk(dev_priv);
 951}
 952
 953static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 954{
 955	drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 956
 957	/*
 958	 * We always enable DPLL0 with the lowest link rate possible, but still
 959	 * taking into account the VCO required to operate the eDP panel at the
 960	 * desired frequency. The usual DP link rates operate with a VCO of
 961	 * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
 962	 * The modeset code is responsible for the selection of the exact link
 963	 * rate later on, with the constraint of choosing a frequency that
 964	 * works with vco.
 965	 */
 966	if (vco == 8640000)
 967		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
 968	else
 969		return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
 970}
 971
 972static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 973{
 974	intel_de_rmw(dev_priv, DPLL_CTRL1,
 975		     DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 976		     DPLL_CTRL1_SSC(SKL_DPLL0) |
 977		     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
 978		     DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
 979		     skl_dpll0_link_rate(dev_priv, vco));
 980	intel_de_posting_read(dev_priv, DPLL_CTRL1);
 981
 982	intel_de_rmw(dev_priv, LCPLL1_CTL,
 983		     0, LCPLL_PLL_ENABLE);
 984
 985	if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
 986		drm_err(&dev_priv->drm, "DPLL0 not locked\n");
 987
 988	dev_priv->cdclk.hw.vco = vco;
 989
 990	/* We'll want to keep using the current vco from now on. */
 991	skl_set_preferred_cdclk_vco(dev_priv, vco);
 992}
 993
 994static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
 995{
 996	intel_de_rmw(dev_priv, LCPLL1_CTL,
 997		     LCPLL_PLL_ENABLE, 0);
 998
 999	if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1000		drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1001
1002	dev_priv->cdclk.hw.vco = 0;
1003}
1004
1005static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1006			      int cdclk, int vco)
1007{
1008	switch (cdclk) {
1009	default:
1010		drm_WARN_ON(&dev_priv->drm,
1011			    cdclk != dev_priv->cdclk.hw.bypass);
1012		drm_WARN_ON(&dev_priv->drm, vco != 0);
1013		fallthrough;
1014	case 308571:
1015	case 337500:
1016		return CDCLK_FREQ_337_308;
1017	case 450000:
1018	case 432000:
1019		return CDCLK_FREQ_450_432;
1020	case 540000:
1021		return CDCLK_FREQ_540;
1022	case 617143:
1023	case 675000:
1024		return CDCLK_FREQ_675_617;
1025	}
1026}
1027
1028static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1029			  const struct intel_cdclk_config *cdclk_config,
1030			  enum pipe pipe)
1031{
1032	int cdclk = cdclk_config->cdclk;
1033	int vco = cdclk_config->vco;
1034	u32 freq_select, cdclk_ctl;
1035	int ret;
1036
1037	/*
1038	 * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1039	 * unsupported on SKL. In theory this should never happen since only
1040	 * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1041	 * supported on SKL either, see the above WA. WARN whenever trying to
1042	 * use the corresponding VCO freq as that always leads to using the
1043	 * minimum 308MHz CDCLK.
1044	 */
1045	drm_WARN_ON_ONCE(&dev_priv->drm,
1046			 IS_SKYLAKE(dev_priv) && vco == 8640000);
1047
1048	ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1049				SKL_CDCLK_PREPARE_FOR_CHANGE,
1050				SKL_CDCLK_READY_FOR_CHANGE,
1051				SKL_CDCLK_READY_FOR_CHANGE, 3);
1052	if (ret) {
1053		drm_err(&dev_priv->drm,
1054			"Failed to inform PCU about cdclk change (%d)\n", ret);
1055		return;
1056	}
1057
1058	freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1059
1060	if (dev_priv->cdclk.hw.vco != 0 &&
1061	    dev_priv->cdclk.hw.vco != vco)
1062		skl_dpll0_disable(dev_priv);
1063
1064	cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1065
1066	if (dev_priv->cdclk.hw.vco != vco) {
1067		/* Wa Display #1183: skl,kbl,cfl */
1068		cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1069		cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1070		intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1071	}
1072
1073	/* Wa Display #1183: skl,kbl,cfl */
1074	cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1075	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1076	intel_de_posting_read(dev_priv, CDCLK_CTL);
1077
1078	if (dev_priv->cdclk.hw.vco != vco)
1079		skl_dpll0_enable(dev_priv, vco);
1080
1081	/* Wa Display #1183: skl,kbl,cfl */
1082	cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1083	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1084
1085	cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1086	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1087
1088	/* Wa Display #1183: skl,kbl,cfl */
1089	cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1090	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1091	intel_de_posting_read(dev_priv, CDCLK_CTL);
1092
1093	/* inform PCU of the change */
1094	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1095				cdclk_config->voltage_level);
1096
1097	intel_update_cdclk(dev_priv);
1098}
1099
1100static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1101{
1102	u32 cdctl, expected;
1103
1104	/*
1105	 * check if the pre-os initialized the display
1106	 * There is SWF18 scratchpad register defined which is set by the
1107	 * pre-os which can be used by the OS drivers to check the status
1108	 */
1109	if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1110		goto sanitize;
1111
1112	intel_update_cdclk(dev_priv);
1113	intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1114
1115	/* Is PLL enabled and locked ? */
1116	if (dev_priv->cdclk.hw.vco == 0 ||
1117	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1118		goto sanitize;
1119
1120	/* DPLL okay; verify the cdclock
1121	 *
1122	 * Noticed in some instances that the freq selection is correct but
1123	 * decimal part is programmed wrong from BIOS where pre-os does not
1124	 * enable display. Verify the same as well.
1125	 */
1126	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1127	expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1128		skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
1129	if (cdctl == expected)
1130		/* All well; nothing to sanitize */
1131		return;
1132
1133sanitize:
1134	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1135
1136	/* force cdclk programming */
1137	dev_priv->cdclk.hw.cdclk = 0;
1138	/* force full PLL disable + enable */
1139	dev_priv->cdclk.hw.vco = -1;
1140}
1141
1142static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1143{
1144	struct intel_cdclk_config cdclk_config;
1145
1146	skl_sanitize_cdclk(dev_priv);
1147
1148	if (dev_priv->cdclk.hw.cdclk != 0 &&
1149	    dev_priv->cdclk.hw.vco != 0) {
1150		/*
1151		 * Use the current vco as our initial
1152		 * guess as to what the preferred vco is.
1153		 */
1154		if (dev_priv->skl_preferred_vco_freq == 0)
1155			skl_set_preferred_cdclk_vco(dev_priv,
1156						    dev_priv->cdclk.hw.vco);
1157		return;
1158	}
1159
1160	cdclk_config = dev_priv->cdclk.hw;
1161
1162	cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1163	if (cdclk_config.vco == 0)
1164		cdclk_config.vco = 8100000;
1165	cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1166	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1167
1168	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1169}
1170
1171static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1172{
1173	struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1174
1175	cdclk_config.cdclk = cdclk_config.bypass;
1176	cdclk_config.vco = 0;
1177	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1178
1179	skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1180}
1181
1182static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1183	{ .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1184	{ .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1185	{ .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1186	{ .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1187	{ .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1188	{}
1189};
1190
1191static const struct intel_cdclk_vals glk_cdclk_table[] = {
1192	{ .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1193	{ .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1194	{ .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1195	{}
1196};
1197
1198static const struct intel_cdclk_vals cnl_cdclk_table[] = {
1199	{ .refclk = 19200, .cdclk = 168000, .divider = 4, .ratio = 35 },
1200	{ .refclk = 19200, .cdclk = 336000, .divider = 2, .ratio = 35 },
1201	{ .refclk = 19200, .cdclk = 528000, .divider = 2, .ratio = 55 },
1202
1203	{ .refclk = 24000, .cdclk = 168000, .divider = 4, .ratio = 28 },
1204	{ .refclk = 24000, .cdclk = 336000, .divider = 2, .ratio = 28 },
1205	{ .refclk = 24000, .cdclk = 528000, .divider = 2, .ratio = 44 },
1206	{}
1207};
1208
1209static const struct intel_cdclk_vals icl_cdclk_table[] = {
1210	{ .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1211	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1212	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1213	{ .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1214	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1215	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1216
1217	{ .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1218	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1219	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1220	{ .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1221	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1222	{ .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1223
1224	{ .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1225	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1226	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1227	{ .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1228	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1229	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1230	{}
1231};
1232
1233static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1234	{ .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1235	{ .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1236	{ .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1237	{ .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1238	{ .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1239	{ .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1240
1241	{ .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1242	{ .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1243	{ .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1244	{ .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1245	{ .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1246	{ .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1247
1248	{ .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1249	{ .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1250	{ .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1251	{ .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1252	{ .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1253	{ .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1254	{}
1255};
1256
1257static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1258	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1259	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1260	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1261
1262	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1263	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1264	{ .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1265
1266	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1267	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1268	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1269	{}
1270};
1271
1272static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1273	{ .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1274	{ .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1275	{ .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1276	{ .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1277	{ .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1278
1279	{ .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1280	{ .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1281	{ .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1282	{ .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1283	{ .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1284
1285	{ .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1286	{ .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1287	{ .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1288	{ .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1289	{ .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1290	{}
1291};
1292
1293static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1294{
1295	const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1296	int i;
1297
1298	for (i = 0; table[i].refclk; i++)
1299		if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1300		    table[i].cdclk >= min_cdclk)
1301			return table[i].cdclk;
1302
1303	drm_WARN(&dev_priv->drm, 1,
1304		 "Cannot satisfy minimum cdclk %d with refclk %u\n",
1305		 min_cdclk, dev_priv->cdclk.hw.ref);
1306	return 0;
1307}
1308
1309static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1310{
1311	const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1312	int i;
1313
1314	if (cdclk == dev_priv->cdclk.hw.bypass)
1315		return 0;
1316
1317	for (i = 0; table[i].refclk; i++)
1318		if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1319		    table[i].cdclk == cdclk)
1320			return dev_priv->cdclk.hw.ref * table[i].ratio;
1321
1322	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1323		 cdclk, dev_priv->cdclk.hw.ref);
1324	return 0;
1325}
1326
1327static u8 bxt_calc_voltage_level(int cdclk)
1328{
1329	return DIV_ROUND_UP(cdclk, 25000);
1330}
1331
1332static u8 cnl_calc_voltage_level(int cdclk)
1333{
1334	if (cdclk > 336000)
1335		return 2;
1336	else if (cdclk > 168000)
1337		return 1;
1338	else
1339		return 0;
1340}
1341
1342static u8 icl_calc_voltage_level(int cdclk)
1343{
1344	if (cdclk > 556800)
1345		return 2;
1346	else if (cdclk > 312000)
1347		return 1;
1348	else
1349		return 0;
1350}
1351
1352static u8 ehl_calc_voltage_level(int cdclk)
1353{
1354	if (cdclk > 326400)
1355		return 3;
1356	else if (cdclk > 312000)
1357		return 2;
1358	else if (cdclk > 180000)
1359		return 1;
1360	else
1361		return 0;
1362}
1363
1364static u8 tgl_calc_voltage_level(int cdclk)
1365{
1366	if (cdclk > 556800)
1367		return 3;
1368	else if (cdclk > 326400)
1369		return 2;
1370	else if (cdclk > 312000)
1371		return 1;
1372	else
1373		return 0;
1374}
1375
1376static void cnl_readout_refclk(struct drm_i915_private *dev_priv,
1377			       struct intel_cdclk_config *cdclk_config)
1378{
1379	if (intel_de_read(dev_priv, SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz)
1380		cdclk_config->ref = 24000;
1381	else
1382		cdclk_config->ref = 19200;
1383}
1384
1385static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1386			       struct intel_cdclk_config *cdclk_config)
1387{
1388	u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1389
1390	switch (dssm) {
1391	default:
1392		MISSING_CASE(dssm);
1393		fallthrough;
1394	case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1395		cdclk_config->ref = 24000;
1396		break;
1397	case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1398		cdclk_config->ref = 19200;
1399		break;
1400	case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1401		cdclk_config->ref = 38400;
1402		break;
1403	}
1404}
1405
1406static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1407			       struct intel_cdclk_config *cdclk_config)
1408{
1409	u32 val, ratio;
1410
1411	if (DISPLAY_VER(dev_priv) >= 11)
1412		icl_readout_refclk(dev_priv, cdclk_config);
1413	else if (IS_CANNONLAKE(dev_priv))
1414		cnl_readout_refclk(dev_priv, cdclk_config);
1415	else
1416		cdclk_config->ref = 19200;
1417
1418	val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1419	if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1420	    (val & BXT_DE_PLL_LOCK) == 0) {
1421		/*
1422		 * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1423		 * setting it to zero is a way to signal that.
1424		 */
1425		cdclk_config->vco = 0;
1426		return;
1427	}
1428
1429	/*
1430	 * CNL+ have the ratio directly in the PLL enable register, gen9lp had
1431	 * it in a separate PLL control register.
1432	 */
1433	if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1434		ratio = val & CNL_CDCLK_PLL_RATIO_MASK;
1435	else
1436		ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1437
1438	cdclk_config->vco = ratio * cdclk_config->ref;
1439}
1440
1441static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1442			  struct intel_cdclk_config *cdclk_config)
1443{
1444	u32 divider;
1445	int div;
1446
1447	bxt_de_pll_readout(dev_priv, cdclk_config);
1448
1449	if (DISPLAY_VER(dev_priv) >= 12)
1450		cdclk_config->bypass = cdclk_config->ref / 2;
1451	else if (DISPLAY_VER(dev_priv) >= 11)
1452		cdclk_config->bypass = 50000;
1453	else
1454		cdclk_config->bypass = cdclk_config->ref;
1455
1456	if (cdclk_config->vco == 0) {
1457		cdclk_config->cdclk = cdclk_config->bypass;
1458		goto out;
1459	}
1460
1461	divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1462
1463	switch (divider) {
1464	case BXT_CDCLK_CD2X_DIV_SEL_1:
1465		div = 2;
1466		break;
1467	case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1468		div = 3;
1469		break;
1470	case BXT_CDCLK_CD2X_DIV_SEL_2:
1471		div = 4;
1472		break;
1473	case BXT_CDCLK_CD2X_DIV_SEL_4:
1474		div = 8;
1475		break;
1476	default:
1477		MISSING_CASE(divider);
1478		return;
1479	}
1480
1481	cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1482
1483 out:
1484	/*
1485	 * Can't read this out :( Let's assume it's
1486	 * at least what the CDCLK frequency requires.
1487	 */
1488	cdclk_config->voltage_level =
1489		dev_priv->display.calc_voltage_level(cdclk_config->cdclk);
1490}
1491
1492static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1493{
1494	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1495
1496	/* Timeout 200us */
1497	if (intel_de_wait_for_clear(dev_priv,
1498				    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1499		drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1500
1501	dev_priv->cdclk.hw.vco = 0;
1502}
1503
1504static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1505{
1506	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1507
1508	intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1509		     BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1510
1511	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1512
1513	/* Timeout 200us */
1514	if (intel_de_wait_for_set(dev_priv,
1515				  BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1516		drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1517
1518	dev_priv->cdclk.hw.vco = vco;
1519}
1520
1521static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1522{
1523	intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1524		     BXT_DE_PLL_PLL_ENABLE, 0);
1525
1526	/* Timeout 200us */
1527	if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1528		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1529
1530	dev_priv->cdclk.hw.vco = 0;
1531}
1532
1533static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1534{
1535	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1536	u32 val;
1537
1538	val = CNL_CDCLK_PLL_RATIO(ratio);
1539	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1540
1541	val |= BXT_DE_PLL_PLL_ENABLE;
1542	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1543
1544	/* Timeout 200us */
1545	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1546		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1547
1548	dev_priv->cdclk.hw.vco = vco;
1549}
1550
1551static bool has_cdclk_crawl(struct drm_i915_private *i915)
1552{
1553	return INTEL_INFO(i915)->has_cdclk_crawl;
1554}
1555
1556static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1557{
1558	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1559	u32 val;
1560
1561	/* Write PLL ratio without disabling */
1562	val = CNL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1563	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1564
1565	/* Submit freq change request */
1566	val |= BXT_DE_PLL_FREQ_REQ;
1567	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1568
1569	/* Timeout 200us */
1570	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1571				  BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1572		DRM_ERROR("timeout waiting for FREQ change request ack\n");
1573
1574	val &= ~BXT_DE_PLL_FREQ_REQ;
1575	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1576
1577	dev_priv->cdclk.hw.vco = vco;
1578}
1579
1580static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1581{
1582	if (DISPLAY_VER(dev_priv) >= 12) {
1583		if (pipe == INVALID_PIPE)
1584			return TGL_CDCLK_CD2X_PIPE_NONE;
1585		else
1586			return TGL_CDCLK_CD2X_PIPE(pipe);
1587	} else if (DISPLAY_VER(dev_priv) >= 11) {
1588		if (pipe == INVALID_PIPE)
1589			return ICL_CDCLK_CD2X_PIPE_NONE;
1590		else
1591			return ICL_CDCLK_CD2X_PIPE(pipe);
1592	} else {
1593		if (pipe == INVALID_PIPE)
1594			return BXT_CDCLK_CD2X_PIPE_NONE;
1595		else
1596			return BXT_CDCLK_CD2X_PIPE(pipe);
1597	}
1598}
1599
1600static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1601				  int cdclk, int vco)
1602{
1603	/* cdclk = vco / 2 / div{1,1.5,2,4} */
1604	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1605	default:
1606		drm_WARN_ON(&dev_priv->drm,
1607			    cdclk != dev_priv->cdclk.hw.bypass);
1608		drm_WARN_ON(&dev_priv->drm, vco != 0);
1609		fallthrough;
1610	case 2:
1611		return BXT_CDCLK_CD2X_DIV_SEL_1;
1612	case 3:
1613		return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1614	case 4:
1615		return BXT_CDCLK_CD2X_DIV_SEL_2;
1616	case 8:
1617		return BXT_CDCLK_CD2X_DIV_SEL_4;
1618	}
1619}
1620
1621static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1622			  const struct intel_cdclk_config *cdclk_config,
1623			  enum pipe pipe)
1624{
1625	int cdclk = cdclk_config->cdclk;
1626	int vco = cdclk_config->vco;
1627	u32 val;
1628	int ret;
1629
1630	/* Inform power controller of upcoming frequency change. */
1631	if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1632		ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1633					SKL_CDCLK_PREPARE_FOR_CHANGE,
1634					SKL_CDCLK_READY_FOR_CHANGE,
1635					SKL_CDCLK_READY_FOR_CHANGE, 3);
1636	else
1637		/*
1638		 * BSpec requires us to wait up to 150usec, but that leads to
1639		 * timeouts; the 2ms used here is based on experiment.
1640		 */
1641		ret = sandybridge_pcode_write_timeout(dev_priv,
1642						      HSW_PCODE_DE_WRITE_FREQ_REQ,
1643						      0x80000000, 150, 2);
1644
1645	if (ret) {
1646		drm_err(&dev_priv->drm,
1647			"Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1648			ret, cdclk);
1649		return;
1650	}
1651
1652	if (has_cdclk_crawl(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) {
1653		if (dev_priv->cdclk.hw.vco != vco)
1654			adlp_cdclk_pll_crawl(dev_priv, vco);
1655	} else if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) {
1656		if (dev_priv->cdclk.hw.vco != 0 &&
1657		    dev_priv->cdclk.hw.vco != vco)
1658			cnl_cdclk_pll_disable(dev_priv);
1659
1660		if (dev_priv->cdclk.hw.vco != vco)
1661			cnl_cdclk_pll_enable(dev_priv, vco);
1662	} else {
1663		if (dev_priv->cdclk.hw.vco != 0 &&
1664		    dev_priv->cdclk.hw.vco != vco)
1665			bxt_de_pll_disable(dev_priv);
1666
1667		if (dev_priv->cdclk.hw.vco != vco)
1668			bxt_de_pll_enable(dev_priv, vco);
1669	}
1670
1671	val = bxt_cdclk_cd2x_div_sel(dev_priv, cdclk, vco) |
1672		bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1673		skl_cdclk_decimal(cdclk);
1674
1675	/*
1676	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
1677	 * enable otherwise.
1678	 */
1679	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1680	    cdclk >= 500000)
1681		val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1682	intel_de_write(dev_priv, CDCLK_CTL, val);
1683
1684	if (pipe != INVALID_PIPE)
1685		intel_wait_for_vblank(dev_priv, pipe);
1686
1687	if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) {
1688		ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1689					      cdclk_config->voltage_level);
1690	} else {
1691		/*
1692		 * The timeout isn't specified, the 2ms used here is based on
1693		 * experiment.
1694		 * FIXME: Waiting for the request completion could be delayed
1695		 * until the next PCODE request based on BSpec.
1696		 */
1697		ret = sandybridge_pcode_write_timeout(dev_priv,
1698						      HSW_PCODE_DE_WRITE_FREQ_REQ,
1699						      cdclk_config->voltage_level,
1700						      150, 2);
1701	}
1702
1703	if (ret) {
1704		drm_err(&dev_priv->drm,
1705			"PCode CDCLK freq set failed, (err %d, freq %d)\n",
1706			ret, cdclk);
1707		return;
1708	}
1709
1710	intel_update_cdclk(dev_priv);
1711
1712	if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1713		/*
1714		 * Can't read out the voltage level :(
1715		 * Let's just assume everything is as expected.
1716		 */
1717		dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
1718}
1719
1720static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1721{
1722	u32 cdctl, expected;
1723	int cdclk, vco;
1724
1725	intel_update_cdclk(dev_priv);
1726	intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1727
1728	if (dev_priv->cdclk.hw.vco == 0 ||
1729	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1730		goto sanitize;
1731
1732	/* DPLL okay; verify the cdclock
1733	 *
1734	 * Some BIOS versions leave an incorrect decimal frequency value and
1735	 * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1736	 * so sanitize this register.
1737	 */
1738	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1739	/*
1740	 * Let's ignore the pipe field, since BIOS could have configured the
1741	 * dividers both synching to an active pipe, or asynchronously
1742	 * (PIPE_NONE).
1743	 */
1744	cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1745
1746	/* Make sure this is a legal cdclk value for the platform */
1747	cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
1748	if (cdclk != dev_priv->cdclk.hw.cdclk)
1749		goto sanitize;
1750
1751	/* Make sure the VCO is correct for the cdclk */
1752	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1753	if (vco != dev_priv->cdclk.hw.vco)
1754		goto sanitize;
1755
1756	expected = skl_cdclk_decimal(cdclk);
1757
1758	/* Figure out what CD2X divider we should be using for this cdclk */
1759	expected |= bxt_cdclk_cd2x_div_sel(dev_priv,
1760					   dev_priv->cdclk.hw.cdclk,
1761					   dev_priv->cdclk.hw.vco);
1762
1763	/*
1764	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
1765	 * enable otherwise.
1766	 */
1767	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1768	    dev_priv->cdclk.hw.cdclk >= 500000)
1769		expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1770
1771	if (cdctl == expected)
1772		/* All well; nothing to sanitize */
1773		return;
1774
1775sanitize:
1776	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1777
1778	/* force cdclk programming */
1779	dev_priv->cdclk.hw.cdclk = 0;
1780
1781	/* force full PLL disable + enable */
1782	dev_priv->cdclk.hw.vco = -1;
1783}
1784
1785static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1786{
1787	struct intel_cdclk_config cdclk_config;
1788
1789	bxt_sanitize_cdclk(dev_priv);
1790
1791	if (dev_priv->cdclk.hw.cdclk != 0 &&
1792	    dev_priv->cdclk.hw.vco != 0)
1793		return;
1794
1795	cdclk_config = dev_priv->cdclk.hw;
1796
1797	/*
1798	 * FIXME:
1799	 * - The initial CDCLK needs to be read from VBT.
1800	 *   Need to make this change after VBT has changes for BXT.
1801	 */
1802	cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1803	cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1804	cdclk_config.voltage_level =
1805		dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1806
1807	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1808}
1809
1810static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1811{
1812	struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1813
1814	cdclk_config.cdclk = cdclk_config.bypass;
1815	cdclk_config.vco = 0;
1816	cdclk_config.voltage_level =
1817		dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1818
1819	bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1820}
1821
1822/**
1823 * intel_cdclk_init_hw - Initialize CDCLK hardware
1824 * @i915: i915 device
1825 *
1826 * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
1827 * sanitizing the state of the hardware if needed. This is generally done only
1828 * during the display core initialization sequence, after which the DMC will
1829 * take care of turning CDCLK off/on as needed.
1830 */
1831void intel_cdclk_init_hw(struct drm_i915_private *i915)
1832{
1833	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1834		bxt_cdclk_init_hw(i915);
1835	else if (DISPLAY_VER(i915) == 9)
1836		skl_cdclk_init_hw(i915);
1837}
1838
1839/**
1840 * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1841 * @i915: i915 device
1842 *
1843 * Uninitialize CDCLK. This is done only during the display core
1844 * uninitialization sequence.
1845 */
1846void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1847{
1848	if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1849		bxt_cdclk_uninit_hw(i915);
1850	else if (DISPLAY_VER(i915) == 9)
1851		skl_cdclk_uninit_hw(i915);
1852}
1853
1854static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
1855				  const struct intel_cdclk_config *a,
1856				  const struct intel_cdclk_config *b)
1857{
1858	int a_div, b_div;
1859
1860	if (!has_cdclk_crawl(dev_priv))
1861		return false;
1862
1863	/*
1864	 * The vco and cd2x divider will change independently
1865	 * from each, so we disallow cd2x change when crawling.
1866	 */
1867	a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
1868	b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
1869
1870	return a->vco != 0 && b->vco != 0 &&
1871		a->vco != b->vco &&
1872		a_div == b_div &&
1873		a->ref == b->ref;
1874}
1875
1876/**
1877 * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1878 *                             configurations requires a modeset on all pipes
1879 * @a: first CDCLK configuration
1880 * @b: second CDCLK configuration
1881 *
1882 * Returns:
1883 * True if changing between the two CDCLK configurations
1884 * requires all pipes to be off, false if not.
1885 */
1886bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1887			       const struct intel_cdclk_config *b)
1888{
1889	return a->cdclk != b->cdclk ||
1890		a->vco != b->vco ||
1891		a->ref != b->ref;
1892}
1893
1894/**
1895 * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
1896 *                               configurations requires only a cd2x divider update
1897 * @dev_priv: i915 device
1898 * @a: first CDCLK configuration
1899 * @b: second CDCLK configuration
1900 *
1901 * Returns:
1902 * True if changing between the two CDCLK configurations
1903 * can be done with just a cd2x divider update, false if not.
1904 */
1905static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
1906					const struct intel_cdclk_config *a,
1907					const struct intel_cdclk_config *b)
1908{
1909	/* Older hw doesn't have the capability */
1910	if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
1911		return false;
1912
1913	return a->cdclk != b->cdclk &&
1914		a->vco == b->vco &&
1915		a->ref == b->ref;
1916}
1917
1918/**
1919 * intel_cdclk_changed - Determine if two CDCLK configurations are different
1920 * @a: first CDCLK configuration
1921 * @b: second CDCLK configuration
1922 *
1923 * Returns:
1924 * True if the CDCLK configurations don't match, false if they do.
1925 */
1926static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
1927				const struct intel_cdclk_config *b)
1928{
1929	return intel_cdclk_needs_modeset(a, b) ||
1930		a->voltage_level != b->voltage_level;
1931}
1932
1933void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config,
1934			     const char *context)
1935{
1936	DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
1937			 context, cdclk_config->cdclk, cdclk_config->vco,
1938			 cdclk_config->ref, cdclk_config->bypass,
1939			 cdclk_config->voltage_level);
1940}
1941
1942/**
1943 * intel_set_cdclk - Push the CDCLK configuration to the hardware
1944 * @dev_priv: i915 device
1945 * @cdclk_config: new CDCLK configuration
1946 * @pipe: pipe with which to synchronize the update
1947 *
1948 * Program the hardware based on the passed in CDCLK state,
1949 * if necessary.
1950 */
1951static void intel_set_cdclk(struct drm_i915_private *dev_priv,
1952			    const struct intel_cdclk_config *cdclk_config,
1953			    enum pipe pipe)
1954{
1955	struct intel_encoder *encoder;
1956
1957	if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
1958		return;
1959
1960	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.set_cdclk))
1961		return;
1962
1963	intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to");
1964
1965	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
1966		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1967
1968		intel_psr_pause(intel_dp);
1969	}
1970
1971	/*
1972	 * Lock aux/gmbus while we change cdclk in case those
1973	 * functions use cdclk. Not all platforms/ports do,
1974	 * but we'll lock them all for simplicity.
1975	 */
1976	mutex_lock(&dev_priv->gmbus_mutex);
1977	for_each_intel_dp(&dev_priv->drm, encoder) {
1978		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1979
1980		mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
1981				     &dev_priv->gmbus_mutex);
1982	}
1983
1984	dev_priv->display.set_cdclk(dev_priv, cdclk_config, pipe);
1985
1986	for_each_intel_dp(&dev_priv->drm, encoder) {
1987		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1988
1989		mutex_unlock(&intel_dp->aux.hw_mutex);
1990	}
1991	mutex_unlock(&dev_priv->gmbus_mutex);
1992
1993	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
1994		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1995
1996		intel_psr_resume(intel_dp);
1997	}
1998
1999	if (drm_WARN(&dev_priv->drm,
2000		     intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
2001		     "cdclk state doesn't match!\n")) {
2002		intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]");
2003		intel_dump_cdclk_config(cdclk_config, "[sw state]");
2004	}
2005}
2006
2007/**
2008 * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2009 * @state: intel atomic state
2010 *
2011 * Program the hardware before updating the HW plane state based on the
2012 * new CDCLK state, if necessary.
2013 */
2014void
2015intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2016{
2017	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2018	const struct intel_cdclk_state *old_cdclk_state =
2019		intel_atomic_get_old_cdclk_state(state);
2020	const struct intel_cdclk_state *new_cdclk_state =
2021		intel_atomic_get_new_cdclk_state(state);
2022	enum pipe pipe = new_cdclk_state->pipe;
2023
2024	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2025				 &new_cdclk_state->actual))
2026		return;
2027
2028	if (pipe == INVALID_PIPE ||
2029	    old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2030		drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2031
2032		intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2033	}
2034}
2035
2036/**
2037 * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2038 * @state: intel atomic state
2039 *
2040 * Program the hardware after updating the HW plane state based on the
2041 * new CDCLK state, if necessary.
2042 */
2043void
2044intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2045{
2046	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2047	const struct intel_cdclk_state *old_cdclk_state =
2048		intel_atomic_get_old_cdclk_state(state);
2049	const struct intel_cdclk_state *new_cdclk_state =
2050		intel_atomic_get_new_cdclk_state(state);
2051	enum pipe pipe = new_cdclk_state->pipe;
2052
2053	if (!intel_cdclk_changed(&old_cdclk_state->actual,
2054				 &new_cdclk_state->actual))
2055		return;
2056
2057	if (pipe != INVALID_PIPE &&
2058	    old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2059		drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2060
2061		intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2062	}
2063}
2064
2065static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2066{
2067	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2068	int pixel_rate = crtc_state->pixel_rate;
2069
2070	if (DISPLAY_VER(dev_priv) >= 10)
2071		return DIV_ROUND_UP(pixel_rate, 2);
2072	else if (DISPLAY_VER(dev_priv) == 9 ||
2073		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2074		return pixel_rate;
2075	else if (IS_CHERRYVIEW(dev_priv))
2076		return DIV_ROUND_UP(pixel_rate * 100, 95);
2077	else if (crtc_state->double_wide)
2078		return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2079	else
2080		return DIV_ROUND_UP(pixel_rate * 100, 90);
2081}
2082
2083static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2084{
2085	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2086	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2087	struct intel_plane *plane;
2088	int min_cdclk = 0;
2089
2090	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2091		min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2092
2093	return min_cdclk;
2094}
2095
2096int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2097{
2098	struct drm_i915_private *dev_priv =
2099		to_i915(crtc_state->uapi.crtc->dev);
2100	int min_cdclk;
2101
2102	if (!crtc_state->hw.enable)
2103		return 0;
2104
2105	min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2106
2107	/* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2108	if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2109		min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2110
2111	/* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2112	 * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2113	 * there may be audio corruption or screen corruption." This cdclk
2114	 * restriction for GLK is 316.8 MHz.
2115	 */
2116	if (intel_crtc_has_dp_encoder(crtc_state) &&
2117	    crtc_state->has_audio &&
2118	    crtc_state->port_clock >= 540000 &&
2119	    crtc_state->lane_count == 4) {
2120		if (DISPLAY_VER(dev_priv) == 10) {
2121			/* Display WA #1145: glk,cnl */
2122			min_cdclk = max(316800, min_cdclk);
2123		} else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2124			/* Display WA #1144: skl,bxt */
2125			min_cdclk = max(432000, min_cdclk);
2126		}
2127	}
2128
2129	/*
2130	 * According to BSpec, "The CD clock frequency must be at least twice
2131	 * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2132	 */
2133	if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2134		min_cdclk = max(2 * 96000, min_cdclk);
2135
2136	/*
2137	 * "For DP audio configuration, cdclk frequency shall be set to
2138	 *  meet the following requirements:
2139	 *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2140	 *  270                    | 320 or higher
2141	 *  162                    | 200 or higher"
2142	 */
2143	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2144	    intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2145		min_cdclk = max(crtc_state->port_clock, min_cdclk);
2146
2147	/*
2148	 * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2149	 * than 320000KHz.
2150	 */
2151	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2152	    IS_VALLEYVIEW(dev_priv))
2153		min_cdclk = max(320000, min_cdclk);
2154
2155	/*
2156	 * On Geminilake once the CDCLK gets as low as 79200
2157	 * picture gets unstable, despite that values are
2158	 * correct for DSI PLL and DE PLL.
2159	 */
2160	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2161	    IS_GEMINILAKE(dev_priv))
2162		min_cdclk = max(158400, min_cdclk);
2163
2164	/* Account for additional needs from the planes */
2165	min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2166
2167	/*
2168	 * HACK. Currently for TGL platforms we calculate
2169	 * min_cdclk initially based on pixel_rate divided
2170	 * by 2, accounting for also plane requirements,
2171	 * however in some cases the lowest possible CDCLK
2172	 * doesn't work and causing the underruns.
2173	 * Explicitly stating here that this seems to be currently
2174	 * rather a Hack, than final solution.
2175	 */
2176	if (IS_TIGERLAKE(dev_priv)) {
2177		/*
2178		 * Clamp to max_cdclk_freq in case pixel rate is higher,
2179		 * in order not to break an 8K, but still leave W/A at place.
2180		 */
2181		min_cdclk = max_t(int, min_cdclk,
2182				  min_t(int, crtc_state->pixel_rate,
2183					dev_priv->max_cdclk_freq));
2184	}
2185
2186	if (min_cdclk > dev_priv->max_cdclk_freq) {
2187		drm_dbg_kms(&dev_priv->drm,
2188			    "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2189			    min_cdclk, dev_priv->max_cdclk_freq);
2190		return -EINVAL;
2191	}
2192
2193	return min_cdclk;
2194}
2195
2196static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2197{
2198	struct intel_atomic_state *state = cdclk_state->base.state;
2199	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2200	struct intel_bw_state *bw_state = NULL;
2201	struct intel_crtc *crtc;
2202	struct intel_crtc_state *crtc_state;
2203	int min_cdclk, i;
2204	enum pipe pipe;
2205
2206	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2207		int ret;
2208
2209		min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2210		if (min_cdclk < 0)
2211			return min_cdclk;
2212
2213		bw_state = intel_atomic_get_bw_state(state);
2214		if (IS_ERR(bw_state))
2215			return PTR_ERR(bw_state);
2216
2217		if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2218			continue;
2219
2220		cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2221
2222		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2223		if (ret)
2224			return ret;
2225	}
2226
2227	min_cdclk = cdclk_state->force_min_cdclk;
2228	for_each_pipe(dev_priv, pipe) {
2229		min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2230
2231		if (!bw_state)
2232			continue;
2233
2234		min_cdclk = max(bw_state->min_cdclk, min_cdclk);
2235	}
2236
2237	return min_cdclk;
2238}
2239
2240/*
2241 * Account for port clock min voltage level requirements.
2242 * This only really does something on CNL+ but can be
2243 * called on earlier platforms as well.
2244 *
2245 * Note that this functions assumes that 0 is
2246 * the lowest voltage value, and higher values
2247 * correspond to increasingly higher voltages.
2248 *
2249 * Should that relationship no longer hold on
2250 * future platforms this code will need to be
2251 * adjusted.
2252 */
2253static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2254{
2255	struct intel_atomic_state *state = cdclk_state->base.state;
2256	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2257	struct intel_crtc *crtc;
2258	struct intel_crtc_state *crtc_state;
2259	u8 min_voltage_level;
2260	int i;
2261	enum pipe pipe;
2262
2263	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2264		int ret;
2265
2266		if (crtc_state->hw.enable)
2267			min_voltage_level = crtc_state->min_voltage_level;
2268		else
2269			min_voltage_level = 0;
2270
2271		if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2272			continue;
2273
2274		cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2275
2276		ret = intel_atomic_lock_global_state(&cdclk_state->base);
2277		if (ret)
2278			return ret;
2279	}
2280
2281	min_voltage_level = 0;
2282	for_each_pipe(dev_priv, pipe)
2283		min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2284					min_voltage_level);
2285
2286	return min_voltage_level;
2287}
2288
2289static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2290{
2291	struct intel_atomic_state *state = cdclk_state->base.state;
2292	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2293	int min_cdclk, cdclk;
2294
2295	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2296	if (min_cdclk < 0)
2297		return min_cdclk;
2298
2299	cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2300
2301	cdclk_state->logical.cdclk = cdclk;
2302	cdclk_state->logical.voltage_level =
2303		vlv_calc_voltage_level(dev_priv, cdclk);
2304
2305	if (!cdclk_state->active_pipes) {
2306		cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2307
2308		cdclk_state->actual.cdclk = cdclk;
2309		cdclk_state->actual.voltage_level =
2310			vlv_calc_voltage_level(dev_priv, cdclk);
2311	} else {
2312		cdclk_state->actual = cdclk_state->logical;
2313	}
2314
2315	return 0;
2316}
2317
2318static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2319{
2320	int min_cdclk, cdclk;
2321
2322	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2323	if (min_cdclk < 0)
2324		return min_cdclk;
2325
2326	/*
2327	 * FIXME should also account for plane ratio
2328	 * once 64bpp pixel formats are supported.
2329	 */
2330	cdclk = bdw_calc_cdclk(min_cdclk);
2331
2332	cdclk_state->logical.cdclk = cdclk;
2333	cdclk_state->logical.voltage_level =
2334		bdw_calc_voltage_level(cdclk);
2335
2336	if (!cdclk_state->active_pipes) {
2337		cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2338
2339		cdclk_state->actual.cdclk = cdclk;
2340		cdclk_state->actual.voltage_level =
2341			bdw_calc_voltage_level(cdclk);
2342	} else {
2343		cdclk_state->actual = cdclk_state->logical;
2344	}
2345
2346	return 0;
2347}
2348
2349static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2350{
2351	struct intel_atomic_state *state = cdclk_state->base.state;
2352	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2353	struct intel_crtc *crtc;
2354	struct intel_crtc_state *crtc_state;
2355	int vco, i;
2356
2357	vco = cdclk_state->logical.vco;
2358	if (!vco)
2359		vco = dev_priv->skl_preferred_vco_freq;
2360
2361	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2362		if (!crtc_state->hw.enable)
2363			continue;
2364
2365		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2366			continue;
2367
2368		/*
2369		 * DPLL0 VCO may need to be adjusted to get the correct
2370		 * clock for eDP. This will affect cdclk as well.
2371		 */
2372		switch (crtc_state->port_clock / 2) {
2373		case 108000:
2374		case 216000:
2375			vco = 8640000;
2376			break;
2377		default:
2378			vco = 8100000;
2379			break;
2380		}
2381	}
2382
2383	return vco;
2384}
2385
2386static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2387{
2388	int min_cdclk, cdclk, vco;
2389
2390	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2391	if (min_cdclk < 0)
2392		return min_cdclk;
2393
2394	vco = skl_dpll0_vco(cdclk_state);
2395
2396	/*
2397	 * FIXME should also account for plane ratio
2398	 * once 64bpp pixel formats are supported.
2399	 */
2400	cdclk = skl_calc_cdclk(min_cdclk, vco);
2401
2402	cdclk_state->logical.vco = vco;
2403	cdclk_state->logical.cdclk = cdclk;
2404	cdclk_state->logical.voltage_level =
2405		skl_calc_voltage_level(cdclk);
2406
2407	if (!cdclk_state->active_pipes) {
2408		cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2409
2410		cdclk_state->actual.vco = vco;
2411		cdclk_state->actual.cdclk = cdclk;
2412		cdclk_state->actual.voltage_level =
2413			skl_calc_voltage_level(cdclk);
2414	} else {
2415		cdclk_state->actual = cdclk_state->logical;
2416	}
2417
2418	return 0;
2419}
2420
2421static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2422{
2423	struct intel_atomic_state *state = cdclk_state->base.state;
2424	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2425	int min_cdclk, min_voltage_level, cdclk, vco;
2426
2427	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2428	if (min_cdclk < 0)
2429		return min_cdclk;
2430
2431	min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2432	if (min_voltage_level < 0)
2433		return min_voltage_level;
2434
2435	cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2436	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2437
2438	cdclk_state->logical.vco = vco;
2439	cdclk_state->logical.cdclk = cdclk;
2440	cdclk_state->logical.voltage_level =
2441		max_t(int, min_voltage_level,
2442		      dev_priv->display.calc_voltage_level(cdclk));
2443
2444	if (!cdclk_state->active_pipes) {
2445		cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2446		vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2447
2448		cdclk_state->actual.vco = vco;
2449		cdclk_state->actual.cdclk = cdclk;
2450		cdclk_state->actual.voltage_level =
2451			dev_priv->display.calc_voltage_level(cdclk);
2452	} else {
2453		cdclk_state->actual = cdclk_state->logical;
2454	}
2455
2456	return 0;
2457}
2458
2459static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2460{
2461	int min_cdclk;
2462
2463	/*
2464	 * We can't change the cdclk frequency, but we still want to
2465	 * check that the required minimum frequency doesn't exceed
2466	 * the actual cdclk frequency.
2467	 */
2468	min_cdclk = intel_compute_min_cdclk(cdclk_state);
2469	if (min_cdclk < 0)
2470		return min_cdclk;
2471
2472	return 0;
2473}
2474
2475static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2476{
2477	struct intel_cdclk_state *cdclk_state;
2478
2479	cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2480	if (!cdclk_state)
2481		return NULL;
2482
2483	cdclk_state->pipe = INVALID_PIPE;
2484
2485	return &cdclk_state->base;
2486}
2487
2488static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2489				      struct intel_global_state *state)
2490{
2491	kfree(state);
2492}
2493
2494static const struct intel_global_state_funcs intel_cdclk_funcs = {
2495	.atomic_duplicate_state = intel_cdclk_duplicate_state,
2496	.atomic_destroy_state = intel_cdclk_destroy_state,
2497};
2498
2499struct intel_cdclk_state *
2500intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2501{
2502	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2503	struct intel_global_state *cdclk_state;
2504
2505	cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj);
2506	if (IS_ERR(cdclk_state))
2507		return ERR_CAST(cdclk_state);
2508
2509	return to_intel_cdclk_state(cdclk_state);
2510}
2511
2512int intel_cdclk_init(struct drm_i915_private *dev_priv)
2513{
2514	struct intel_cdclk_state *cdclk_state;
2515
2516	cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2517	if (!cdclk_state)
2518		return -ENOMEM;
2519
2520	intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj,
2521				     &cdclk_state->base, &intel_cdclk_funcs);
2522
2523	return 0;
2524}
2525
2526int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2527{
2528	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2529	const struct intel_cdclk_state *old_cdclk_state;
2530	struct intel_cdclk_state *new_cdclk_state;
2531	enum pipe pipe = INVALID_PIPE;
2532	int ret;
2533
2534	new_cdclk_state = intel_atomic_get_cdclk_state(state);
2535	if (IS_ERR(new_cdclk_state))
2536		return PTR_ERR(new_cdclk_state);
2537
2538	old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2539
2540	new_cdclk_state->active_pipes =
2541		intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2542
2543	ret = dev_priv->display.modeset_calc_cdclk(new_cdclk_state);
2544	if (ret)
2545		return ret;
2546
2547	if (intel_cdclk_changed(&old_cdclk_state->actual,
2548				&new_cdclk_state->actual)) {
2549		/*
2550		 * Also serialize commits across all crtcs
2551		 * if the actual hw needs to be poked.
2552		 */
2553		ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2554		if (ret)
2555			return ret;
2556	} else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2557		   old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2558		   intel_cdclk_changed(&old_cdclk_state->logical,
2559				       &new_cdclk_state->logical)) {
2560		ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2561		if (ret)
2562			return ret;
2563	} else {
2564		return 0;
2565	}
2566
2567	if (is_power_of_2(new_cdclk_state->active_pipes) &&
2568	    intel_cdclk_can_cd2x_update(dev_priv,
2569					&old_cdclk_state->actual,
2570					&new_cdclk_state->actual)) {
2571		struct intel_crtc *crtc;
2572		struct intel_crtc_state *crtc_state;
2573
2574		pipe = ilog2(new_cdclk_state->active_pipes);
2575		crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
2576
2577		crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2578		if (IS_ERR(crtc_state))
2579			return PTR_ERR(crtc_state);
2580
2581		if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2582			pipe = INVALID_PIPE;
2583	}
2584
2585	if (intel_cdclk_can_crawl(dev_priv,
2586				  &old_cdclk_state->actual,
2587				  &new_cdclk_state->actual)) {
2588		drm_dbg_kms(&dev_priv->drm,
2589			    "Can change cdclk via crawl\n");
2590	} else if (pipe != INVALID_PIPE) {
2591		new_cdclk_state->pipe = pipe;
2592
2593		drm_dbg_kms(&dev_priv->drm,
2594			    "Can change cdclk cd2x divider with pipe %c active\n",
2595			    pipe_name(pipe));
2596	} else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2597					     &new_cdclk_state->actual)) {
2598		/* All pipes must be switched off while we change the cdclk. */
2599		ret = intel_modeset_all_pipes(state);
2600		if (ret)
2601			return ret;
2602
2603		drm_dbg_kms(&dev_priv->drm,
2604			    "Modeset required for cdclk change\n");
2605	}
2606
2607	drm_dbg_kms(&dev_priv->drm,
2608		    "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2609		    new_cdclk_state->logical.cdclk,
2610		    new_cdclk_state->actual.cdclk);
2611	drm_dbg_kms(&dev_priv->drm,
2612		    "New voltage level calculated to be logical %u, actual %u\n",
2613		    new_cdclk_state->logical.voltage_level,
2614		    new_cdclk_state->actual.voltage_level);
2615
2616	return 0;
2617}
2618
2619static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2620{
2621	int max_cdclk_freq = dev_priv->max_cdclk_freq;
2622
2623	if (DISPLAY_VER(dev_priv) >= 10)
2624		return 2 * max_cdclk_freq;
2625	else if (DISPLAY_VER(dev_priv) == 9 ||
2626		 IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2627		return max_cdclk_freq;
2628	else if (IS_CHERRYVIEW(dev_priv))
2629		return max_cdclk_freq*95/100;
2630	else if (DISPLAY_VER(dev_priv) < 4)
2631		return 2*max_cdclk_freq*90/100;
2632	else
2633		return max_cdclk_freq*90/100;
2634}
2635
2636/**
2637 * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2638 * @dev_priv: i915 device
2639 *
2640 * Determine the maximum CDCLK frequency the platform supports, and also
2641 * derive the maximum dot clock frequency the maximum CDCLK frequency
2642 * allows.
2643 */
2644void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2645{
2646	if (IS_JSL_EHL(dev_priv)) {
2647		if (dev_priv->cdclk.hw.ref == 24000)
2648			dev_priv->max_cdclk_freq = 552000;
2649		else
2650			dev_priv->max_cdclk_freq = 556800;
2651	} else if (DISPLAY_VER(dev_priv) >= 11) {
2652		if (dev_priv->cdclk.hw.ref == 24000)
2653			dev_priv->max_cdclk_freq = 648000;
2654		else
2655			dev_priv->max_cdclk_freq = 652800;
2656	} else if (IS_CANNONLAKE(dev_priv)) {
2657		dev_priv->max_cdclk_freq = 528000;
2658	} else if (IS_GEMINILAKE(dev_priv)) {
2659		dev_priv->max_cdclk_freq = 316800;
2660	} else if (IS_BROXTON(dev_priv)) {
2661		dev_priv->max_cdclk_freq = 624000;
2662	} else if (DISPLAY_VER(dev_priv) == 9) {
2663		u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2664		int max_cdclk, vco;
2665
2666		vco = dev_priv->skl_preferred_vco_freq;
2667		drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2668
2669		/*
2670		 * Use the lower (vco 8640) cdclk values as a
2671		 * first guess. skl_calc_cdclk() will correct it
2672		 * if the preferred vco is 8100 instead.
2673		 */
2674		if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2675			max_cdclk = 617143;
2676		else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2677			max_cdclk = 540000;
2678		else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2679			max_cdclk = 432000;
2680		else
2681			max_cdclk = 308571;
2682
2683		dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2684	} else if (IS_BROADWELL(dev_priv))  {
2685		/*
2686		 * FIXME with extra cooling we can allow
2687		 * 540 MHz for ULX and 675 Mhz for ULT.
2688		 * How can we know if extra cooling is
2689		 * available? PCI ID, VTB, something else?
2690		 */
2691		if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2692			dev_priv->max_cdclk_freq = 450000;
2693		else if (IS_BDW_ULX(dev_priv))
2694			dev_priv->max_cdclk_freq = 450000;
2695		else if (IS_BDW_ULT(dev_priv))
2696			dev_priv->max_cdclk_freq = 540000;
2697		else
2698			dev_priv->max_cdclk_freq = 675000;
2699	} else if (IS_CHERRYVIEW(dev_priv)) {
2700		dev_priv->max_cdclk_freq = 320000;
2701	} else if (IS_VALLEYVIEW(dev_priv)) {
2702		dev_priv->max_cdclk_freq = 400000;
2703	} else {
2704		/* otherwise assume cdclk is fixed */
2705		dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
2706	}
2707
2708	dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2709
2710	drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2711		dev_priv->max_cdclk_freq);
2712
2713	drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2714		dev_priv->max_dotclk_freq);
2715}
2716
2717/**
2718 * intel_update_cdclk - Determine the current CDCLK frequency
2719 * @dev_priv: i915 device
2720 *
2721 * Determine the current CDCLK frequency.
2722 */
2723void intel_update_cdclk(struct drm_i915_private *dev_priv)
2724{
2725	dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw);
2726
2727	/*
2728	 * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2729	 * Programmng [sic] note: bit[9:2] should be programmed to the number
2730	 * of cdclk that generates 4MHz reference clock freq which is used to
2731	 * generate GMBus clock. This will vary with the cdclk freq.
2732	 */
2733	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2734		intel_de_write(dev_priv, GMBUSFREQ_VLV,
2735			       DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
2736}
2737
2738static int dg1_rawclk(struct drm_i915_private *dev_priv)
2739{
2740	/*
2741	 * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2742	 * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2743	 */
2744	intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2745		       CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2746
2747	return 38400;
2748}
2749
2750static int cnp_rawclk(struct drm_i915_private *dev_priv)
2751{
2752	u32 rawclk;
2753	int divider, fraction;
2754
2755	if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2756		/* 24 MHz */
2757		divider = 24000;
2758		fraction = 0;
2759	} else {
2760		/* 19.2 MHz */
2761		divider = 19000;
2762		fraction = 200;
2763	}
2764
2765	rawclk = CNP_RAWCLK_DIV(divider / 1000);
2766	if (fraction) {
2767		int numerator = 1;
2768
2769		rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2770							   fraction) - 1);
2771		if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2772			rawclk |= ICP_RAWCLK_NUM(numerator);
2773	}
2774
2775	intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2776	return divider + fraction;
2777}
2778
2779static int pch_rawclk(struct drm_i915_private *dev_priv)
2780{
2781	return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2782}
2783
2784static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2785{
2786	/* RAWCLK_FREQ_VLV register updated from power well code */
2787	return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2788				      CCK_DISPLAY_REF_CLOCK_CONTROL);
2789}
2790
2791static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2792{
2793	u32 clkcfg;
2794
2795	/*
2796	 * hrawclock is 1/4 the FSB frequency
2797	 *
2798	 * Note that this only reads the state of the FSB
2799	 * straps, not the actual FSB frequency. Some BIOSen
2800	 * let you configure each independently. Ideally we'd
2801	 * read out the actual FSB frequency but sadly we
2802	 * don't know which registers have that information,
2803	 * and all the relevant docs have gone to bit heaven :(
2804	 */
2805	clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2806
2807	if (IS_MOBILE(dev_priv)) {
2808		switch (clkcfg) {
2809		case CLKCFG_FSB_400:
2810			return 100000;
2811		case CLKCFG_FSB_533:
2812			return 133333;
2813		case CLKCFG_FSB_667:
2814			return 166667;
2815		case CLKCFG_FSB_800:
2816			return 200000;
2817		case CLKCFG_FSB_1067:
2818			return 266667;
2819		case CLKCFG_FSB_1333:
2820			return 333333;
2821		default:
2822			MISSING_CASE(clkcfg);
2823			return 133333;
2824		}
2825	} else {
2826		switch (clkcfg) {
2827		case CLKCFG_FSB_400_ALT:
2828			return 100000;
2829		case CLKCFG_FSB_533:
2830			return 133333;
2831		case CLKCFG_FSB_667:
2832			return 166667;
2833		case CLKCFG_FSB_800:
2834			return 200000;
2835		case CLKCFG_FSB_1067_ALT:
2836			return 266667;
2837		case CLKCFG_FSB_1333_ALT:
2838			return 333333;
2839		case CLKCFG_FSB_1600_ALT:
2840			return 400000;
2841		default:
2842			return 133333;
2843		}
2844	}
2845}
2846
2847/**
2848 * intel_read_rawclk - Determine the current RAWCLK frequency
2849 * @dev_priv: i915 device
2850 *
2851 * Determine the current RAWCLK frequency. RAWCLK is a fixed
2852 * frequency clock so this needs to done only once.
2853 */
2854u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
2855{
2856	u32 freq;
2857
2858	if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
2859		freq = dg1_rawclk(dev_priv);
2860	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
2861		freq = cnp_rawclk(dev_priv);
2862	else if (HAS_PCH_SPLIT(dev_priv))
2863		freq = pch_rawclk(dev_priv);
2864	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2865		freq = vlv_hrawclk(dev_priv);
2866	else if (DISPLAY_VER(dev_priv) >= 3)
2867		freq = i9xx_hrawclk(dev_priv);
2868	else
2869		/* no rawclk on other platforms, or no need to know it */
2870		return 0;
2871
2872	return freq;
2873}
2874
2875/**
2876 * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
2877 * @dev_priv: i915 device
2878 */
2879void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
2880{
2881	if (IS_ALDERLAKE_P(dev_priv)) {
2882		dev_priv->display.set_cdclk = bxt_set_cdclk;
2883		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2884		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2885		dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2886		/* Wa_22011320316:adlp[a0] */
2887		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0))
2888			dev_priv->cdclk.table = adlp_a_step_cdclk_table;
2889		else
2890			dev_priv->cdclk.table = adlp_cdclk_table;
2891	} else if (IS_ROCKETLAKE(dev_priv)) {
2892		dev_priv->display.set_cdclk = bxt_set_cdclk;
2893		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2894		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2895		dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2896		dev_priv->cdclk.table = rkl_cdclk_table;
2897	} else if (DISPLAY_VER(dev_priv) >= 12) {
2898		dev_priv->display.set_cdclk = bxt_set_cdclk;
2899		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2900		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2901		dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2902		dev_priv->cdclk.table = icl_cdclk_table;
2903	} else if (IS_JSL_EHL(dev_priv)) {
2904		dev_priv->display.set_cdclk = bxt_set_cdclk;
2905		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2906		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2907		dev_priv->display.calc_voltage_level = ehl_calc_voltage_level;
2908		dev_priv->cdclk.table = icl_cdclk_table;
2909	} else if (DISPLAY_VER(dev_priv) >= 11) {
2910		dev_priv->display.set_cdclk = bxt_set_cdclk;
2911		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2912		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2913		dev_priv->display.calc_voltage_level = icl_calc_voltage_level;
2914		dev_priv->cdclk.table = icl_cdclk_table;
2915	} else if (IS_CANNONLAKE(dev_priv)) {
2916		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2917		dev_priv->display.set_cdclk = bxt_set_cdclk;
2918		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2919		dev_priv->display.calc_voltage_level = cnl_calc_voltage_level;
2920		dev_priv->cdclk.table = cnl_cdclk_table;
2921	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
2922		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2923		dev_priv->display.set_cdclk = bxt_set_cdclk;
2924		dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2925		dev_priv->display.calc_voltage_level = bxt_calc_voltage_level;
2926		if (IS_GEMINILAKE(dev_priv))
2927			dev_priv->cdclk.table = glk_cdclk_table;
2928		else
2929			dev_priv->cdclk.table = bxt_cdclk_table;
2930	} else if (DISPLAY_VER(dev_priv) == 9) {
2931		dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2932		dev_priv->display.set_cdclk = skl_set_cdclk;
2933		dev_priv->display.modeset_calc_cdclk = skl_modeset_calc_cdclk;
2934	} else if (IS_BROADWELL(dev_priv)) {
2935		dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2936		dev_priv->display.set_cdclk = bdw_set_cdclk;
2937		dev_priv->display.modeset_calc_cdclk = bdw_modeset_calc_cdclk;
2938	} else if (IS_CHERRYVIEW(dev_priv)) {
2939		dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2940		dev_priv->display.set_cdclk = chv_set_cdclk;
2941		dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2942	} else if (IS_VALLEYVIEW(dev_priv)) {
2943		dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2944		dev_priv->display.set_cdclk = vlv_set_cdclk;
2945		dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2946	} else {
2947		dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2948		dev_priv->display.modeset_calc_cdclk = fixed_modeset_calc_cdclk;
2949	}
2950
2951	if (DISPLAY_VER(dev_priv) >= 10 || IS_BROXTON(dev_priv))
2952		dev_priv->display.get_cdclk = bxt_get_cdclk;
2953	else if (DISPLAY_VER(dev_priv) == 9)
2954		dev_priv->display.get_cdclk = skl_get_cdclk;
2955	else if (IS_BROADWELL(dev_priv))
2956		dev_priv->display.get_cdclk = bdw_get_cdclk;
2957	else if (IS_HASWELL(dev_priv))
2958		dev_priv->display.get_cdclk = hsw_get_cdclk;
2959	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2960		dev_priv->display.get_cdclk = vlv_get_cdclk;
2961	else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv))
2962		dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2963	else if (IS_IRONLAKE(dev_priv))
2964		dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk;
2965	else if (IS_GM45(dev_priv))
2966		dev_priv->display.get_cdclk = gm45_get_cdclk;
2967	else if (IS_G45(dev_priv))
2968		dev_priv->display.get_cdclk = g33_get_cdclk;
2969	else if (IS_I965GM(dev_priv))
2970		dev_priv->display.get_cdclk = i965gm_get_cdclk;
2971	else if (IS_I965G(dev_priv))
2972		dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2973	else if (IS_PINEVIEW(dev_priv))
2974		dev_priv->display.get_cdclk = pnv_get_cdclk;
2975	else if (IS_G33(dev_priv))
2976		dev_priv->display.get_cdclk = g33_get_cdclk;
2977	else if (IS_I945GM(dev_priv))
2978		dev_priv->display.get_cdclk = i945gm_get_cdclk;
2979	else if (IS_I945G(dev_priv))
2980		dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2981	else if (IS_I915GM(dev_priv))
2982		dev_priv->display.get_cdclk = i915gm_get_cdclk;
2983	else if (IS_I915G(dev_priv))
2984		dev_priv->display.get_cdclk = fixed_333mhz_get_cdclk;
2985	else if (IS_I865G(dev_priv))
2986		dev_priv->display.get_cdclk = fixed_266mhz_get_cdclk;
2987	else if (IS_I85X(dev_priv))
2988		dev_priv->display.get_cdclk = i85x_get_cdclk;
2989	else if (IS_I845G(dev_priv))
2990		dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk;
2991	else if (IS_I830(dev_priv))
2992		dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk;
2993
2994	if (drm_WARN(&dev_priv->drm, !dev_priv->display.get_cdclk,
2995		     "Unknown platform. Assuming 133 MHz CDCLK\n"))
2996		dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk;
2997}