1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/pci.h>
  25#include <linux/seq_file.h>
  26
  27#include "r600_dpm.h"
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "trinity_dpm.h"
  31#include "trinityd.h"
  32#include "vce.h"
  33
  34#define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
  35#define TRINITY_MINIMUM_ENGINE_CLOCK 800
  36#define SCLK_MIN_DIV_INTV_SHIFT     12
  37#define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
  38
  39#ifndef TRINITY_MGCG_SEQUENCE
  40#define TRINITY_MGCG_SEQUENCE  100
  41
  42static const u32 trinity_mgcg_shls_default[] =
  43{
  44	/* Register, Value, Mask */
  45	0x0000802c, 0xc0000000, 0xffffffff,
  46	0x00003fc4, 0xc0000000, 0xffffffff,
  47	0x00005448, 0x00000100, 0xffffffff,
  48	0x000055e4, 0x00000100, 0xffffffff,
  49	0x0000160c, 0x00000100, 0xffffffff,
  50	0x00008984, 0x06000100, 0xffffffff,
  51	0x0000c164, 0x00000100, 0xffffffff,
  52	0x00008a18, 0x00000100, 0xffffffff,
  53	0x0000897c, 0x06000100, 0xffffffff,
  54	0x00008b28, 0x00000100, 0xffffffff,
  55	0x00009144, 0x00800200, 0xffffffff,
  56	0x00009a60, 0x00000100, 0xffffffff,
  57	0x00009868, 0x00000100, 0xffffffff,
  58	0x00008d58, 0x00000100, 0xffffffff,
  59	0x00009510, 0x00000100, 0xffffffff,
  60	0x0000949c, 0x00000100, 0xffffffff,
  61	0x00009654, 0x00000100, 0xffffffff,
  62	0x00009030, 0x00000100, 0xffffffff,
  63	0x00009034, 0x00000100, 0xffffffff,
  64	0x00009038, 0x00000100, 0xffffffff,
  65	0x0000903c, 0x00000100, 0xffffffff,
  66	0x00009040, 0x00000100, 0xffffffff,
  67	0x0000a200, 0x00000100, 0xffffffff,
  68	0x0000a204, 0x00000100, 0xffffffff,
  69	0x0000a208, 0x00000100, 0xffffffff,
  70	0x0000a20c, 0x00000100, 0xffffffff,
  71	0x00009744, 0x00000100, 0xffffffff,
  72	0x00003f80, 0x00000100, 0xffffffff,
  73	0x0000a210, 0x00000100, 0xffffffff,
  74	0x0000a214, 0x00000100, 0xffffffff,
  75	0x000004d8, 0x00000100, 0xffffffff,
  76	0x00009664, 0x00000100, 0xffffffff,
  77	0x00009698, 0x00000100, 0xffffffff,
  78	0x000004d4, 0x00000200, 0xffffffff,
  79	0x000004d0, 0x00000000, 0xffffffff,
  80	0x000030cc, 0x00000104, 0xffffffff,
  81	0x0000d0c0, 0x00000100, 0xffffffff,
  82	0x0000d8c0, 0x00000100, 0xffffffff,
  83	0x0000951c, 0x00010000, 0xffffffff,
  84	0x00009160, 0x00030002, 0xffffffff,
  85	0x00009164, 0x00050004, 0xffffffff,
  86	0x00009168, 0x00070006, 0xffffffff,
  87	0x00009178, 0x00070000, 0xffffffff,
  88	0x0000917c, 0x00030002, 0xffffffff,
  89	0x00009180, 0x00050004, 0xffffffff,
  90	0x0000918c, 0x00010006, 0xffffffff,
  91	0x00009190, 0x00090008, 0xffffffff,
  92	0x00009194, 0x00070000, 0xffffffff,
  93	0x00009198, 0x00030002, 0xffffffff,
  94	0x0000919c, 0x00050004, 0xffffffff,
  95	0x000091a8, 0x00010006, 0xffffffff,
  96	0x000091ac, 0x00090008, 0xffffffff,
  97	0x000091b0, 0x00070000, 0xffffffff,
  98	0x000091b4, 0x00030002, 0xffffffff,
  99	0x000091b8, 0x00050004, 0xffffffff,
 100	0x000091c4, 0x00010006, 0xffffffff,
 101	0x000091c8, 0x00090008, 0xffffffff,
 102	0x000091cc, 0x00070000, 0xffffffff,
 103	0x000091d0, 0x00030002, 0xffffffff,
 104	0x000091d4, 0x00050004, 0xffffffff,
 105	0x000091e0, 0x00010006, 0xffffffff,
 106	0x000091e4, 0x00090008, 0xffffffff,
 107	0x000091e8, 0x00000000, 0xffffffff,
 108	0x000091ec, 0x00070000, 0xffffffff,
 109	0x000091f0, 0x00030002, 0xffffffff,
 110	0x000091f4, 0x00050004, 0xffffffff,
 111	0x00009200, 0x00010006, 0xffffffff,
 112	0x00009204, 0x00090008, 0xffffffff,
 113	0x00009208, 0x00070000, 0xffffffff,
 114	0x0000920c, 0x00030002, 0xffffffff,
 115	0x00009210, 0x00050004, 0xffffffff,
 116	0x0000921c, 0x00010006, 0xffffffff,
 117	0x00009220, 0x00090008, 0xffffffff,
 118	0x00009294, 0x00000000, 0xffffffff
 119};
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 120#endif
 121
 122#ifndef TRINITY_SYSLS_SEQUENCE
 123#define TRINITY_SYSLS_SEQUENCE  100
 124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 125static const u32 trinity_sysls_disable[] =
 126{
 127	/* Register, Value, Mask */
 128	0x0000d0c0, 0x00000000, 0xffffffff,
 129	0x0000d8c0, 0x00000000, 0xffffffff,
 130	0x000055e8, 0x00000000, 0xffffffff,
 131	0x0000d0bc, 0x00000000, 0xffffffff,
 132	0x0000d8bc, 0x00000000, 0xffffffff,
 133	0x000015c0, 0x00041401, 0xffffffff,
 134	0x0000264c, 0x00040400, 0xffffffff,
 135	0x00002648, 0x00040400, 0xffffffff,
 136	0x00002650, 0x00040400, 0xffffffff,
 137	0x000020b8, 0x00040400, 0xffffffff,
 138	0x000020bc, 0x00040400, 0xffffffff,
 139	0x000020c0, 0x00040c80, 0xffffffff,
 140	0x0000f4a0, 0x000000c0, 0xffffffff,
 141	0x0000f4a4, 0x00680000, 0xffffffff,
 142	0x00002f50, 0x00000404, 0xffffffff,
 143	0x000004c8, 0x00000001, 0xffffffff,
 144	0x0000641c, 0x00007ffd, 0xffffffff,
 145	0x00000c7c, 0x0000ff00, 0xffffffff,
 146	0x00006dfc, 0x0000007f, 0xffffffff
 147};
 148
 149static const u32 trinity_sysls_enable[] =
 150{
 151	/* Register, Value, Mask */
 152	0x000055e8, 0x00000001, 0xffffffff,
 153	0x0000d0bc, 0x00000100, 0xffffffff,
 154	0x0000d8bc, 0x00000100, 0xffffffff,
 155	0x000015c0, 0x000c1401, 0xffffffff,
 156	0x0000264c, 0x000c0400, 0xffffffff,
 157	0x00002648, 0x000c0400, 0xffffffff,
 158	0x00002650, 0x000c0400, 0xffffffff,
 159	0x000020b8, 0x000c0400, 0xffffffff,
 160	0x000020bc, 0x000c0400, 0xffffffff,
 161	0x000020c0, 0x000c0c80, 0xffffffff,
 162	0x0000f4a0, 0x000000c0, 0xffffffff,
 163	0x0000f4a4, 0x00680fff, 0xffffffff,
 164	0x00002f50, 0x00000903, 0xffffffff,
 165	0x000004c8, 0x00000000, 0xffffffff,
 166	0x0000641c, 0x00000000, 0xffffffff,
 167	0x00000c7c, 0x00000000, 0xffffffff,
 168	0x00006dfc, 0x00000000, 0xffffffff
 169};
 170#endif
 171
 172static const u32 trinity_override_mgpg_sequences[] =
 173{
 174	/* Register, Value */
 175	0x00000200, 0xE030032C,
 176	0x00000204, 0x00000FFF,
 177	0x00000200, 0xE0300058,
 178	0x00000204, 0x00030301,
 179	0x00000200, 0xE0300054,
 180	0x00000204, 0x500010FF,
 181	0x00000200, 0xE0300074,
 182	0x00000204, 0x00030301,
 183	0x00000200, 0xE0300070,
 184	0x00000204, 0x500010FF,
 185	0x00000200, 0xE0300090,
 186	0x00000204, 0x00030301,
 187	0x00000200, 0xE030008C,
 188	0x00000204, 0x500010FF,
 189	0x00000200, 0xE03000AC,
 190	0x00000204, 0x00030301,
 191	0x00000200, 0xE03000A8,
 192	0x00000204, 0x500010FF,
 193	0x00000200, 0xE03000C8,
 194	0x00000204, 0x00030301,
 195	0x00000200, 0xE03000C4,
 196	0x00000204, 0x500010FF,
 197	0x00000200, 0xE03000E4,
 198	0x00000204, 0x00030301,
 199	0x00000200, 0xE03000E0,
 200	0x00000204, 0x500010FF,
 201	0x00000200, 0xE0300100,
 202	0x00000204, 0x00030301,
 203	0x00000200, 0xE03000FC,
 204	0x00000204, 0x500010FF,
 205	0x00000200, 0xE0300058,
 206	0x00000204, 0x00030303,
 207	0x00000200, 0xE0300054,
 208	0x00000204, 0x600010FF,
 209	0x00000200, 0xE0300074,
 210	0x00000204, 0x00030303,
 211	0x00000200, 0xE0300070,
 212	0x00000204, 0x600010FF,
 213	0x00000200, 0xE0300090,
 214	0x00000204, 0x00030303,
 215	0x00000200, 0xE030008C,
 216	0x00000204, 0x600010FF,
 217	0x00000200, 0xE03000AC,
 218	0x00000204, 0x00030303,
 219	0x00000200, 0xE03000A8,
 220	0x00000204, 0x600010FF,
 221	0x00000200, 0xE03000C8,
 222	0x00000204, 0x00030303,
 223	0x00000200, 0xE03000C4,
 224	0x00000204, 0x600010FF,
 225	0x00000200, 0xE03000E4,
 226	0x00000204, 0x00030303,
 227	0x00000200, 0xE03000E0,
 228	0x00000204, 0x600010FF,
 229	0x00000200, 0xE0300100,
 230	0x00000204, 0x00030303,
 231	0x00000200, 0xE03000FC,
 232	0x00000204, 0x600010FF,
 233	0x00000200, 0xE0300058,
 234	0x00000204, 0x00030303,
 235	0x00000200, 0xE0300054,
 236	0x00000204, 0x700010FF,
 237	0x00000200, 0xE0300074,
 238	0x00000204, 0x00030303,
 239	0x00000200, 0xE0300070,
 240	0x00000204, 0x700010FF,
 241	0x00000200, 0xE0300090,
 242	0x00000204, 0x00030303,
 243	0x00000200, 0xE030008C,
 244	0x00000204, 0x700010FF,
 245	0x00000200, 0xE03000AC,
 246	0x00000204, 0x00030303,
 247	0x00000200, 0xE03000A8,
 248	0x00000204, 0x700010FF,
 249	0x00000200, 0xE03000C8,
 250	0x00000204, 0x00030303,
 251	0x00000200, 0xE03000C4,
 252	0x00000204, 0x700010FF,
 253	0x00000200, 0xE03000E4,
 254	0x00000204, 0x00030303,
 255	0x00000200, 0xE03000E0,
 256	0x00000204, 0x700010FF,
 257	0x00000200, 0xE0300100,
 258	0x00000204, 0x00030303,
 259	0x00000200, 0xE03000FC,
 260	0x00000204, 0x700010FF,
 261	0x00000200, 0xE0300058,
 262	0x00000204, 0x00010303,
 263	0x00000200, 0xE0300054,
 264	0x00000204, 0x800010FF,
 265	0x00000200, 0xE0300074,
 266	0x00000204, 0x00010303,
 267	0x00000200, 0xE0300070,
 268	0x00000204, 0x800010FF,
 269	0x00000200, 0xE0300090,
 270	0x00000204, 0x00010303,
 271	0x00000200, 0xE030008C,
 272	0x00000204, 0x800010FF,
 273	0x00000200, 0xE03000AC,
 274	0x00000204, 0x00010303,
 275	0x00000200, 0xE03000A8,
 276	0x00000204, 0x800010FF,
 277	0x00000200, 0xE03000C4,
 278	0x00000204, 0x800010FF,
 279	0x00000200, 0xE03000C8,
 280	0x00000204, 0x00010303,
 281	0x00000200, 0xE03000E4,
 282	0x00000204, 0x00010303,
 283	0x00000200, 0xE03000E0,
 284	0x00000204, 0x800010FF,
 285	0x00000200, 0xE0300100,
 286	0x00000204, 0x00010303,
 287	0x00000200, 0xE03000FC,
 288	0x00000204, 0x800010FF,
 289	0x00000200, 0x0001f198,
 290	0x00000204, 0x0003ffff,
 291	0x00000200, 0x0001f19C,
 292	0x00000204, 0x3fffffff,
 293	0x00000200, 0xE030032C,
 294	0x00000204, 0x00000000,
 295};
 296
 
 297static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 298						   const u32 *seq, u32 count);
 299static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
 300static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
 301					     struct radeon_ps *new_rps,
 302					     struct radeon_ps *old_rps);
 303
 304static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
 305{
 306	struct trinity_ps *ps = rps->ps_priv;
 307
 308	return ps;
 309}
 310
 311static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
 312{
 313	struct trinity_power_info *pi = rdev->pm.dpm.priv;
 314
 315	return pi;
 316}
 317
 318static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
 319{
 320	struct trinity_power_info *pi = trinity_get_pi(rdev);
 321	u32 p, u;
 322	u32 value;
 323	struct atom_clock_dividers dividers;
 324	u32 xclk = radeon_get_xclk(rdev);
 325	u32 sssd = 1;
 326	int ret;
 327	u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
 328
 329	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 330					     25000, false, &dividers);
 331	if (ret)
 332		return;
 333
 334	value = RREG32_SMC(GFX_POWER_GATING_CNTL);
 335	value &= ~(SSSD_MASK | PDS_DIV_MASK);
 336	if (sssd)
 337		value |= SSSD(1);
 338	value |= PDS_DIV(dividers.post_div);
 339	WREG32_SMC(GFX_POWER_GATING_CNTL, value);
 340
 341	r600_calculate_u_and_p(500, xclk, 16, &p, &u);
 342
 343	WREG32(CG_PG_CTRL, SP(p) | SU(u));
 344
 345	WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
 346
 347	/* XXX double check hw_rev */
 348	if (pi->override_dynamic_mgpg && (hw_rev == 0))
 349		trinity_override_dynamic_mg_powergating(rdev);
 350
 351}
 352
 353#define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
 354#define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
 355#define CGTS_SM_CTRL_REG_DISABLE    0x00600000
 356#define CGTS_SM_CTRL_REG_ENABLE     0x96944200
 357
 358static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
 359					  bool enable)
 360{
 361	u32 local0;
 362	u32 local1;
 363
 364	if (enable) {
 365		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 366		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 367
 368		WREG32_CG(CG_CGTT_LOCAL_0,
 369			  (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 370		WREG32_CG(CG_CGTT_LOCAL_1,
 371			  (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 372
 373		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
 374	} else {
 375		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
 376
 377		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 378		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 379
 380		WREG32_CG(CG_CGTT_LOCAL_0,
 381			  CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 382		WREG32_CG(CG_CGTT_LOCAL_1,
 383			  CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 384	}
 385}
 386
 387static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
 388{
 389	u32 count;
 390	const u32 *seq = NULL;
 391
 392	seq = &trinity_mgcg_shls_default[0];
 393	count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
 394
 395	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 396}
 397
 398static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
 399					   bool enable)
 400{
 401	if (enable) {
 402		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 403	} else {
 404		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 405		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 406		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 407		RREG32(GB_ADDR_CONFIG);
 408	}
 409}
 410
 411static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 412						   const u32 *seq, u32 count)
 413{
 414	u32 i, length = count * 3;
 415
 416	for (i = 0; i < length; i += 3)
 417		WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
 418}
 419
 420static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
 421						    const u32 *seq, u32 count)
 422{
 423	u32  i, length = count * 2;
 424
 425	for (i = 0; i < length; i += 2)
 426		WREG32(seq[i], seq[i+1]);
 427
 428}
 429
 430static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
 431{
 432	u32 count;
 433	const u32 *seq = NULL;
 434
 435	seq = &trinity_override_mgpg_sequences[0];
 436	count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
 437
 438	trinity_program_override_mgpg_sequences(rdev, seq, count);
 439}
 440
 441static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
 442					  bool enable)
 443{
 444	u32 count;
 445	const u32 *seq = NULL;
 446
 447	if (enable) {
 448		seq = &trinity_sysls_enable[0];
 449		count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
 450	} else {
 451		seq = &trinity_sysls_disable[0];
 452		count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
 453	}
 454
 455	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 456}
 457
 458static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
 459					   bool enable)
 460{
 461	if (enable) {
 462		if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
 463			WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
 464
 465		WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
 466	} else {
 467		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
 468		RREG32(GB_ADDR_CONFIG);
 469	}
 470}
 471
 472static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
 473					    bool enable)
 474{
 475	u32 value;
 476
 477	if (enable) {
 478		value = RREG32_SMC(PM_I_CNTL_1);
 479		value &= ~DS_PG_CNTL_MASK;
 480		value |= DS_PG_CNTL(1);
 481		WREG32_SMC(PM_I_CNTL_1, value);
 482
 483		value = RREG32_SMC(SMU_S_PG_CNTL);
 484		value &= ~DS_PG_EN_MASK;
 485		value |= DS_PG_EN(1);
 486		WREG32_SMC(SMU_S_PG_CNTL, value);
 487	} else {
 488		value = RREG32_SMC(SMU_S_PG_CNTL);
 489		value &= ~DS_PG_EN_MASK;
 490		WREG32_SMC(SMU_S_PG_CNTL, value);
 491
 492		value = RREG32_SMC(PM_I_CNTL_1);
 493		value &= ~DS_PG_CNTL_MASK;
 494		WREG32_SMC(PM_I_CNTL_1, value);
 495	}
 496
 497	trinity_gfx_dynamic_mgpg_config(rdev);
 498
 499}
 500
 501static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
 502{
 503	struct trinity_power_info *pi = trinity_get_pi(rdev);
 504
 505	if (pi->enable_gfx_clock_gating)
 506		sumo_gfx_clockgating_initialize(rdev);
 507	if (pi->enable_mg_clock_gating)
 508		trinity_mg_clockgating_initialize(rdev);
 509	if (pi->enable_gfx_power_gating)
 510		trinity_gfx_powergating_initialize(rdev);
 511	if (pi->enable_mg_clock_gating) {
 512		trinity_ls_clockgating_enable(rdev, true);
 513		trinity_mg_clockgating_enable(rdev, true);
 514	}
 515	if (pi->enable_gfx_clock_gating)
 516		trinity_gfx_clockgating_enable(rdev, true);
 517	if (pi->enable_gfx_dynamic_mgpg)
 518		trinity_gfx_dynamic_mgpg_enable(rdev, true);
 519	if (pi->enable_gfx_power_gating)
 520		trinity_gfx_powergating_enable(rdev, true);
 521}
 522
 523static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
 524{
 525	struct trinity_power_info *pi = trinity_get_pi(rdev);
 526
 527	if (pi->enable_gfx_power_gating)
 528		trinity_gfx_powergating_enable(rdev, false);
 529	if (pi->enable_gfx_dynamic_mgpg)
 530		trinity_gfx_dynamic_mgpg_enable(rdev, false);
 531	if (pi->enable_gfx_clock_gating)
 532		trinity_gfx_clockgating_enable(rdev, false);
 533	if (pi->enable_mg_clock_gating) {
 534		trinity_mg_clockgating_enable(rdev, false);
 535		trinity_ls_clockgating_enable(rdev, false);
 536	}
 537}
 538
 539static void trinity_set_divider_value(struct radeon_device *rdev,
 540				      u32 index, u32 sclk)
 541{
 542	struct atom_clock_dividers  dividers;
 543	int ret;
 544	u32 value;
 545	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 546
 547	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 548					     sclk, false, &dividers);
 549	if (ret)
 550		return;
 551
 552	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 553	value &= ~CLK_DIVIDER_MASK;
 554	value |= CLK_DIVIDER(dividers.post_div);
 555	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 556
 557	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 558					     sclk/2, false, &dividers);
 559	if (ret)
 560		return;
 561
 562	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
 563	value &= ~PD_SCLK_DIVIDER_MASK;
 564	value |= PD_SCLK_DIVIDER(dividers.post_div);
 565	WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
 566}
 567
 568static void trinity_set_ds_dividers(struct radeon_device *rdev,
 569				    u32 index, u32 divider)
 570{
 571	u32 value;
 572	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 573
 574	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 575	value &= ~DS_DIV_MASK;
 576	value |= DS_DIV(divider);
 577	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 578}
 579
 580static void trinity_set_ss_dividers(struct radeon_device *rdev,
 581				    u32 index, u32 divider)
 582{
 583	u32 value;
 584	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 585
 586	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 587	value &= ~DS_SH_DIV_MASK;
 588	value |= DS_SH_DIV(divider);
 589	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 590}
 591
 592static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
 593{
 594	struct trinity_power_info *pi = trinity_get_pi(rdev);
 595	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
 596	u32 value;
 597	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 598
 599	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 600	value &= ~VID_MASK;
 601	value |= VID(vid_7bit);
 602	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 603
 604	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 605	value &= ~LVRT_MASK;
 606	value |= LVRT(0);
 607	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 608}
 609
 610static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
 611				       u32 index, u32 gnb_slow)
 612{
 613	u32 value;
 614	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 615
 616	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 617	value &= ~GNB_SLOW_MASK;
 618	value |= GNB_SLOW(gnb_slow);
 619	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 620}
 621
 622static void trinity_set_force_nbp_state(struct radeon_device *rdev,
 623					u32 index, u32 force_nbp_state)
 624{
 625	u32 value;
 626	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 627
 628	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 629	value &= ~FORCE_NBPS1_MASK;
 630	value |= FORCE_NBPS1(force_nbp_state);
 631	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 632}
 633
 634static void trinity_set_display_wm(struct radeon_device *rdev,
 635				   u32 index, u32 wm)
 636{
 637	u32 value;
 638	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 639
 640	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 641	value &= ~DISPLAY_WM_MASK;
 642	value |= DISPLAY_WM(wm);
 643	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 644}
 645
 646static void trinity_set_vce_wm(struct radeon_device *rdev,
 647			       u32 index, u32 wm)
 648{
 649	u32 value;
 650	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 651
 652	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 653	value &= ~VCE_WM_MASK;
 654	value |= VCE_WM(wm);
 655	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 656}
 657
 658static void trinity_set_at(struct radeon_device *rdev,
 659			   u32 index, u32 at)
 660{
 661	u32 value;
 662	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 663
 664	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
 665	value &= ~AT_MASK;
 666	value |= AT(at);
 667	WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
 668}
 669
 670static void trinity_program_power_level(struct radeon_device *rdev,
 671					struct trinity_pl *pl, u32 index)
 672{
 673	struct trinity_power_info *pi = trinity_get_pi(rdev);
 674
 675	if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
 676		return;
 677
 678	trinity_set_divider_value(rdev, index, pl->sclk);
 679	trinity_set_vid(rdev, index, pl->vddc_index);
 680	trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
 681	trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
 682	trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
 683	trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
 684	trinity_set_display_wm(rdev, index, pl->display_wm);
 685	trinity_set_vce_wm(rdev, index, pl->vce_wm);
 686	trinity_set_at(rdev, index, pi->at[index]);
 687}
 688
 689static void trinity_power_level_enable_disable(struct radeon_device *rdev,
 690					       u32 index, bool enable)
 691{
 692	u32 value;
 693	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 694
 695	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 696	value &= ~STATE_VALID_MASK;
 697	if (enable)
 698		value |= STATE_VALID(1);
 699	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 700}
 701
 702static bool trinity_dpm_enabled(struct radeon_device *rdev)
 703{
 704	if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
 705		return true;
 706	else
 707		return false;
 708}
 709
 710static void trinity_start_dpm(struct radeon_device *rdev)
 711{
 712	u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 713
 714	value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
 715	value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
 716	WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
 717
 718	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 719	WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
 720
 721	trinity_dpm_config(rdev, true);
 722}
 723
 724static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
 725{
 726	int i;
 727
 728	for (i = 0; i < rdev->usec_timeout; i++) {
 729		if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
 730			break;
 731		udelay(1);
 732	}
 733	for (i = 0; i < rdev->usec_timeout; i++) {
 734		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
 735			break;
 736		udelay(1);
 737	}
 738	for (i = 0; i < rdev->usec_timeout; i++) {
 739		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 740			break;
 741		udelay(1);
 742	}
 743}
 744
 745static void trinity_stop_dpm(struct radeon_device *rdev)
 746{
 747	u32 sclk_dpm_cntl;
 748
 749	WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
 750
 751	sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 752	sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
 753	WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
 754
 755	trinity_dpm_config(rdev, false);
 756}
 757
 758static void trinity_start_am(struct radeon_device *rdev)
 759{
 760	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 761}
 762
 763static void trinity_reset_am(struct radeon_device *rdev)
 764{
 765	WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
 766		 ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 767}
 768
 769static void trinity_wait_for_level_0(struct radeon_device *rdev)
 770{
 771	int i;
 772
 773	for (i = 0; i < rdev->usec_timeout; i++) {
 774		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 775			break;
 776		udelay(1);
 777	}
 778}
 779
 780static void trinity_enable_power_level_0(struct radeon_device *rdev)
 781{
 782	trinity_power_level_enable_disable(rdev, 0, true);
 783}
 784
 785static void trinity_force_level_0(struct radeon_device *rdev)
 786{
 787	trinity_dpm_force_state(rdev, 0);
 788}
 789
 790static void trinity_unforce_levels(struct radeon_device *rdev)
 791{
 792	trinity_dpm_no_forced_level(rdev);
 793}
 794
 795static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
 796						struct radeon_ps *new_rps,
 797						struct radeon_ps *old_rps)
 798{
 799	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 800	struct trinity_ps *old_ps = trinity_get_ps(old_rps);
 801	u32 i;
 802	u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
 803
 804	for (i = 0; i < new_ps->num_levels; i++) {
 805		trinity_program_power_level(rdev, &new_ps->levels[i], i);
 806		trinity_power_level_enable_disable(rdev, i, true);
 807	}
 808
 809	for (i = new_ps->num_levels; i < n_current_state_levels; i++)
 810		trinity_power_level_enable_disable(rdev, i, false);
 811}
 812
 813static void trinity_program_bootup_state(struct radeon_device *rdev)
 814{
 815	struct trinity_power_info *pi = trinity_get_pi(rdev);
 816	u32 i;
 817
 818	trinity_program_power_level(rdev, &pi->boot_pl, 0);
 819	trinity_power_level_enable_disable(rdev, 0, true);
 820
 821	for (i = 1; i < 8; i++)
 822		trinity_power_level_enable_disable(rdev, i, false);
 823}
 824
 825static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
 826					  struct radeon_ps *rps)
 827{
 828	struct trinity_ps *ps = trinity_get_ps(rps);
 829	u32 uvdstates = (ps->vclk_low_divider |
 830			 ps->vclk_high_divider << 8 |
 831			 ps->dclk_low_divider << 16 |
 832			 ps->dclk_high_divider << 24);
 833
 834	WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
 835}
 836
 837static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
 838					   u32 interval)
 839{
 840	u32 p, u;
 841	u32 tp = RREG32_SMC(PM_TP);
 842	u32 val;
 843	u32 xclk = radeon_get_xclk(rdev);
 844
 845	r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
 846
 847	val = (p + tp - 1) / tp;
 848
 849	WREG32_SMC(SMU_UVD_DPM_CNTL, val);
 850}
 851
 852static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
 853{
 854	if ((rps->vclk == 0) && (rps->dclk == 0))
 855		return true;
 856	else
 857		return false;
 858}
 859
 860static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
 861				     struct radeon_ps *rps2)
 862{
 863	struct trinity_ps *ps1 = trinity_get_ps(rps1);
 864	struct trinity_ps *ps2 = trinity_get_ps(rps2);
 865
 866	if ((rps1->vclk == rps2->vclk) &&
 867	    (rps1->dclk == rps2->dclk) &&
 868	    (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
 869	    (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
 870	    (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
 871	    (ps1->dclk_high_divider == ps2->dclk_high_divider))
 872		return true;
 873	else
 874		return false;
 875}
 876
 877static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
 878				     struct radeon_ps *new_rps,
 879				     struct radeon_ps *old_rps)
 880{
 881	struct trinity_power_info *pi = trinity_get_pi(rdev);
 882
 883	if (pi->enable_gfx_power_gating) {
 884		trinity_gfx_powergating_enable(rdev, false);
 885	}
 886
 887	if (pi->uvd_dpm) {
 888		if (trinity_uvd_clocks_zero(new_rps) &&
 889		    !trinity_uvd_clocks_zero(old_rps)) {
 890			trinity_setup_uvd_dpm_interval(rdev, 0);
 891		} else if (!trinity_uvd_clocks_zero(new_rps)) {
 892			trinity_setup_uvd_clock_table(rdev, new_rps);
 893
 894			if (trinity_uvd_clocks_zero(old_rps)) {
 895				u32 tmp = RREG32(CG_MISC_REG);
 896				tmp &= 0xfffffffd;
 897				WREG32(CG_MISC_REG, tmp);
 898
 899				radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 900
 901				trinity_setup_uvd_dpm_interval(rdev, 3000);
 902			}
 903		}
 904		trinity_uvd_dpm_config(rdev);
 905	} else {
 906		if (trinity_uvd_clocks_zero(new_rps) ||
 907		    trinity_uvd_clocks_equal(new_rps, old_rps))
 908			return;
 909
 910		radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 911	}
 912
 913	if (pi->enable_gfx_power_gating) {
 914		trinity_gfx_powergating_enable(rdev, true);
 915	}
 916}
 917
 918static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
 919						       struct radeon_ps *new_rps,
 920						       struct radeon_ps *old_rps)
 921{
 922	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 923	struct trinity_ps *current_ps = trinity_get_ps(new_rps);
 924
 925	if (new_ps->levels[new_ps->num_levels - 1].sclk >=
 926	    current_ps->levels[current_ps->num_levels - 1].sclk)
 927		return;
 928
 929	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 930}
 931
 932static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
 933						      struct radeon_ps *new_rps,
 934						      struct radeon_ps *old_rps)
 935{
 936	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 937	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
 938
 939	if (new_ps->levels[new_ps->num_levels - 1].sclk <
 940	    current_ps->levels[current_ps->num_levels - 1].sclk)
 941		return;
 942
 943	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 944}
 945
 946static void trinity_set_vce_clock(struct radeon_device *rdev,
 947				  struct radeon_ps *new_rps,
 948				  struct radeon_ps *old_rps)
 949{
 950	if ((old_rps->evclk != new_rps->evclk) ||
 951	    (old_rps->ecclk != new_rps->ecclk)) {
 952		/* turn the clocks on when encoding, off otherwise */
 953		if (new_rps->evclk || new_rps->ecclk)
 954			vce_v1_0_enable_mgcg(rdev, false);
 955		else
 956			vce_v1_0_enable_mgcg(rdev, true);
 957		radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
 958	}
 959}
 960
 961static void trinity_program_ttt(struct radeon_device *rdev)
 962{
 963	struct trinity_power_info *pi = trinity_get_pi(rdev);
 964	u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
 965
 966	value &= ~(HT_MASK | LT_MASK);
 967	value |= HT((pi->thermal_auto_throttling + 49) * 8);
 968	value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
 969	WREG32_SMC(SMU_SCLK_DPM_TTT, value);
 970}
 971
 972static void trinity_enable_att(struct radeon_device *rdev)
 973{
 974	u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
 975
 976	value &= ~SCLK_TT_EN_MASK;
 977	value |= SCLK_TT_EN(1);
 978	WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
 979}
 980
 981static void trinity_program_sclk_dpm(struct radeon_device *rdev)
 982{
 983	u32 p, u;
 984	u32 tp = RREG32_SMC(PM_TP);
 985	u32 ni;
 986	u32 xclk = radeon_get_xclk(rdev);
 987	u32 value;
 988
 989	r600_calculate_u_and_p(400, xclk, 16, &p, &u);
 990
 991	ni = (p + tp - 1) / tp;
 992
 993	value = RREG32_SMC(PM_I_CNTL_1);
 994	value &= ~SCLK_DPM_MASK;
 995	value |= SCLK_DPM(ni);
 996	WREG32_SMC(PM_I_CNTL_1, value);
 997}
 998
 999static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
1000						 int min_temp, int max_temp)
1001{
1002	int low_temp = 0 * 1000;
1003	int high_temp = 255 * 1000;
1004
1005	if (low_temp < min_temp)
1006		low_temp = min_temp;
1007	if (high_temp > max_temp)
1008		high_temp = max_temp;
1009	if (high_temp < low_temp) {
1010		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1011		return -EINVAL;
1012	}
1013
1014	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1015	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1016
1017	rdev->pm.dpm.thermal.min_temp = low_temp;
1018	rdev->pm.dpm.thermal.max_temp = high_temp;
1019
1020	return 0;
1021}
1022
1023static void trinity_update_current_ps(struct radeon_device *rdev,
1024				      struct radeon_ps *rps)
1025{
1026	struct trinity_ps *new_ps = trinity_get_ps(rps);
1027	struct trinity_power_info *pi = trinity_get_pi(rdev);
1028
1029	pi->current_rps = *rps;
1030	pi->current_ps = *new_ps;
1031	pi->current_rps.ps_priv = &pi->current_ps;
1032}
1033
1034static void trinity_update_requested_ps(struct radeon_device *rdev,
1035					struct radeon_ps *rps)
1036{
1037	struct trinity_ps *new_ps = trinity_get_ps(rps);
1038	struct trinity_power_info *pi = trinity_get_pi(rdev);
1039
1040	pi->requested_rps = *rps;
1041	pi->requested_ps = *new_ps;
1042	pi->requested_rps.ps_priv = &pi->requested_ps;
1043}
1044
1045void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1046{
1047	struct trinity_power_info *pi = trinity_get_pi(rdev);
1048
1049	if (pi->enable_bapm) {
1050		trinity_acquire_mutex(rdev);
1051		trinity_dpm_bapm_enable(rdev, enable);
1052		trinity_release_mutex(rdev);
1053	}
1054}
1055
1056int trinity_dpm_enable(struct radeon_device *rdev)
1057{
1058	struct trinity_power_info *pi = trinity_get_pi(rdev);
1059
1060	trinity_acquire_mutex(rdev);
1061
1062	if (trinity_dpm_enabled(rdev)) {
1063		trinity_release_mutex(rdev);
1064		return -EINVAL;
1065	}
1066
1067	trinity_program_bootup_state(rdev);
1068	sumo_program_vc(rdev, 0x00C00033);
1069	trinity_start_am(rdev);
1070	if (pi->enable_auto_thermal_throttling) {
1071		trinity_program_ttt(rdev);
1072		trinity_enable_att(rdev);
1073	}
1074	trinity_program_sclk_dpm(rdev);
1075	trinity_start_dpm(rdev);
1076	trinity_wait_for_dpm_enabled(rdev);
1077	trinity_dpm_bapm_enable(rdev, false);
1078	trinity_release_mutex(rdev);
1079
1080	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1081
1082	return 0;
1083}
1084
1085int trinity_dpm_late_enable(struct radeon_device *rdev)
1086{
1087	int ret;
1088
1089	trinity_acquire_mutex(rdev);
1090	trinity_enable_clock_power_gating(rdev);
1091
1092	if (rdev->irq.installed &&
1093	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1094		ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1095		if (ret) {
1096			trinity_release_mutex(rdev);
1097			return ret;
1098		}
1099		rdev->irq.dpm_thermal = true;
1100		radeon_irq_set(rdev);
1101	}
1102	trinity_release_mutex(rdev);
1103
1104	return 0;
1105}
1106
1107void trinity_dpm_disable(struct radeon_device *rdev)
1108{
1109	trinity_acquire_mutex(rdev);
1110	if (!trinity_dpm_enabled(rdev)) {
1111		trinity_release_mutex(rdev);
1112		return;
1113	}
1114	trinity_dpm_bapm_enable(rdev, false);
1115	trinity_disable_clock_power_gating(rdev);
1116	sumo_clear_vc(rdev);
1117	trinity_wait_for_level_0(rdev);
1118	trinity_stop_dpm(rdev);
1119	trinity_reset_am(rdev);
1120	trinity_release_mutex(rdev);
1121
1122	if (rdev->irq.installed &&
1123	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1124		rdev->irq.dpm_thermal = false;
1125		radeon_irq_set(rdev);
1126	}
1127
1128	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1129}
1130
1131static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1132{
1133	struct trinity_power_info *pi = trinity_get_pi(rdev);
1134
1135	pi->min_sclk_did =
1136		(RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1137}
1138
1139static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1140				  struct radeon_ps *rps)
1141{
1142	struct trinity_power_info *pi = trinity_get_pi(rdev);
1143	struct trinity_ps *new_ps = trinity_get_ps(rps);
1144	u32 nbpsconfig;
1145
1146	if (pi->sys_info.nb_dpm_enable) {
1147		nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1148		nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1149		nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1150			       Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1151			       DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1152			       DpmXNbPsHi(new_ps->DpmXNbPsHi));
1153		WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1154	}
1155}
1156
1157int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1158					enum radeon_dpm_forced_level level)
1159{
1160	struct trinity_power_info *pi = trinity_get_pi(rdev);
1161	struct radeon_ps *rps = &pi->current_rps;
1162	struct trinity_ps *ps = trinity_get_ps(rps);
1163	int i, ret;
1164
1165	if (ps->num_levels <= 1)
1166		return 0;
1167
1168	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1169		/* not supported by the hw */
1170		return -EINVAL;
1171	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1172		ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1173		if (ret)
1174			return ret;
1175	} else {
1176		for (i = 0; i < ps->num_levels; i++) {
1177			ret = trinity_dpm_n_levels_disabled(rdev, 0);
1178			if (ret)
1179				return ret;
1180		}
1181	}
1182
1183	rdev->pm.dpm.forced_level = level;
1184
1185	return 0;
1186}
1187
1188int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1189{
1190	struct trinity_power_info *pi = trinity_get_pi(rdev);
1191	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1192	struct radeon_ps *new_ps = &requested_ps;
1193
1194	trinity_update_requested_ps(rdev, new_ps);
1195
1196	trinity_apply_state_adjust_rules(rdev,
1197					 &pi->requested_rps,
1198					 &pi->current_rps);
1199
1200	return 0;
1201}
1202
1203int trinity_dpm_set_power_state(struct radeon_device *rdev)
1204{
1205	struct trinity_power_info *pi = trinity_get_pi(rdev);
1206	struct radeon_ps *new_ps = &pi->requested_rps;
1207	struct radeon_ps *old_ps = &pi->current_rps;
1208
1209	trinity_acquire_mutex(rdev);
1210	if (pi->enable_dpm) {
1211		if (pi->enable_bapm)
1212			trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1213		trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1214		trinity_enable_power_level_0(rdev);
1215		trinity_force_level_0(rdev);
1216		trinity_wait_for_level_0(rdev);
1217		trinity_setup_nbp_sim(rdev, new_ps);
1218		trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1219		trinity_force_level_0(rdev);
1220		trinity_unforce_levels(rdev);
1221		trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1222		trinity_set_vce_clock(rdev, new_ps, old_ps);
1223	}
1224	trinity_release_mutex(rdev);
1225
1226	return 0;
1227}
1228
1229void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1230{
1231	struct trinity_power_info *pi = trinity_get_pi(rdev);
1232	struct radeon_ps *new_ps = &pi->requested_rps;
1233
1234	trinity_update_current_ps(rdev, new_ps);
1235}
1236
1237void trinity_dpm_setup_asic(struct radeon_device *rdev)
1238{
1239	trinity_acquire_mutex(rdev);
1240	sumo_program_sstp(rdev);
1241	sumo_take_smu_control(rdev, true);
1242	trinity_get_min_sclk_divider(rdev);
1243	trinity_release_mutex(rdev);
1244}
1245
1246#if 0
1247void trinity_dpm_reset_asic(struct radeon_device *rdev)
1248{
1249	struct trinity_power_info *pi = trinity_get_pi(rdev);
1250
1251	trinity_acquire_mutex(rdev);
1252	if (pi->enable_dpm) {
1253		trinity_enable_power_level_0(rdev);
1254		trinity_force_level_0(rdev);
1255		trinity_wait_for_level_0(rdev);
1256		trinity_program_bootup_state(rdev);
1257		trinity_force_level_0(rdev);
1258		trinity_unforce_levels(rdev);
1259	}
1260	trinity_release_mutex(rdev);
1261}
1262#endif
1263
1264static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1265						  u32 vid_2bit)
1266{
1267	struct trinity_power_info *pi = trinity_get_pi(rdev);
1268	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1269	u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1270	u32 step = (svi_mode == 0) ? 1250 : 625;
1271	u32 delta = vid_7bit * step + 50;
1272
1273	if (delta > 155000)
1274		return 0;
1275
1276	return (155000 - delta) / 100;
1277}
1278
1279static void trinity_patch_boot_state(struct radeon_device *rdev,
1280				     struct trinity_ps *ps)
1281{
1282	struct trinity_power_info *pi = trinity_get_pi(rdev);
1283
1284	ps->num_levels = 1;
1285	ps->nbps_flags = 0;
1286	ps->bapm_flags = 0;
1287	ps->levels[0] = pi->boot_pl;
1288}
1289
1290static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1291{
1292	if (sclk < 20000)
1293		return 1;
1294	return 0;
1295}
1296
1297static void trinity_construct_boot_state(struct radeon_device *rdev)
1298{
1299	struct trinity_power_info *pi = trinity_get_pi(rdev);
1300
1301	pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1302	pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1303	pi->boot_pl.ds_divider_index = 0;
1304	pi->boot_pl.ss_divider_index = 0;
1305	pi->boot_pl.allow_gnb_slow = 1;
1306	pi->boot_pl.force_nbp_state = 0;
1307	pi->boot_pl.display_wm = 0;
1308	pi->boot_pl.vce_wm = 0;
1309	pi->current_ps.num_levels = 1;
1310	pi->current_ps.levels[0] = pi->boot_pl;
1311}
1312
1313static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1314						  u32 sclk, u32 min_sclk_in_sr)
1315{
1316	struct trinity_power_info *pi = trinity_get_pi(rdev);
1317	u32 i;
1318	u32 temp;
1319	u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1320		min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1321
1322	if (sclk < min)
1323		return 0;
1324
1325	if (!pi->enable_sclk_ds)
1326		return 0;
1327
1328	for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1329		temp = sclk / sumo_get_sleep_divider_from_id(i);
1330		if (temp >= min || i == 0)
1331			break;
1332	}
1333
1334	return (u8)i;
1335}
1336
1337static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1338					  u32 lower_limit)
1339{
1340	struct trinity_power_info *pi = trinity_get_pi(rdev);
1341	u32 i;
1342
1343	for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1344		if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1345			return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1346	}
1347
1348	if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1349		DRM_ERROR("engine clock out of range!");
1350
1351	return 0;
1352}
1353
1354static void trinity_patch_thermal_state(struct radeon_device *rdev,
1355					struct trinity_ps *ps,
1356					struct trinity_ps *current_ps)
1357{
1358	struct trinity_power_info *pi = trinity_get_pi(rdev);
1359	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1360	u32 current_vddc;
1361	u32 current_sclk;
1362	u32 current_index = 0;
1363
1364	if (current_ps) {
1365		current_vddc = current_ps->levels[current_index].vddc_index;
1366		current_sclk = current_ps->levels[current_index].sclk;
1367	} else {
1368		current_vddc = pi->boot_pl.vddc_index;
1369		current_sclk = pi->boot_pl.sclk;
1370	}
1371
1372	ps->levels[0].vddc_index = current_vddc;
1373
1374	if (ps->levels[0].sclk > current_sclk)
1375		ps->levels[0].sclk = current_sclk;
1376
1377	ps->levels[0].ds_divider_index =
1378		trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1379	ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1380	ps->levels[0].allow_gnb_slow = 1;
1381	ps->levels[0].force_nbp_state = 0;
1382	ps->levels[0].display_wm = 0;
1383	ps->levels[0].vce_wm =
1384		trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1385}
1386
1387static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1388				       struct trinity_ps *ps, u32 index)
1389{
1390	if (ps == NULL || ps->num_levels <= 1)
1391		return 0;
1392	else if (ps->num_levels == 2) {
1393		if (index == 0)
1394			return 0;
1395		else
1396			return 1;
1397	} else {
1398		if (index == 0)
1399			return 0;
1400		else if (ps->levels[index].sclk < 30000)
1401			return 0;
1402		else
1403			return 1;
1404	}
1405}
1406
1407static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1408				       struct radeon_ps *rps)
1409{
1410	struct trinity_power_info *pi = trinity_get_pi(rdev);
1411	u32 i = 0;
1412
1413	for (i = 0; i < 4; i++) {
1414		if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1415		    (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1416		    break;
1417	}
1418
1419	if (i >= 4) {
1420		DRM_ERROR("UVD clock index not found!\n");
1421		i = 3;
1422	}
1423	return i;
1424}
1425
1426static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1427				     struct radeon_ps *rps)
1428{
1429	struct trinity_ps *ps = trinity_get_ps(rps);
1430	struct trinity_power_info *pi = trinity_get_pi(rdev);
1431	u32 high_index = 0;
1432	u32 low_index = 0;
1433
1434	if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1435		high_index = trinity_get_uvd_clock_index(rdev, rps);
1436
1437		switch(high_index) {
1438		case 3:
1439		case 2:
1440			low_index = 1;
1441			break;
1442		case 1:
1443		case 0:
1444		default:
1445			low_index = 0;
1446			break;
1447		}
1448
1449		ps->vclk_low_divider =
1450			pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1451		ps->dclk_low_divider =
1452			pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1453		ps->vclk_high_divider =
1454			pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1455		ps->dclk_high_divider =
1456			pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1457	}
1458}
1459
1460static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1461					 u32 evclk, u32 ecclk, u16 *voltage)
1462{
1463	u32 i;
1464	int ret = -EINVAL;
1465	struct radeon_vce_clock_voltage_dependency_table *table =
1466		&rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1467
1468	if (((evclk == 0) && (ecclk == 0)) ||
1469	    (table && (table->count == 0))) {
1470		*voltage = 0;
1471		return 0;
1472	}
1473
1474	for (i = 0; i < table->count; i++) {
1475		if ((evclk <= table->entries[i].evclk) &&
1476		    (ecclk <= table->entries[i].ecclk)) {
1477			*voltage = table->entries[i].v;
1478			ret = 0;
1479			break;
1480		}
1481	}
1482
1483	/* if no match return the highest voltage */
1484	if (ret)
1485		*voltage = table->entries[table->count - 1].v;
1486
1487	return ret;
1488}
1489
1490static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1491					     struct radeon_ps *new_rps,
1492					     struct radeon_ps *old_rps)
1493{
1494	struct trinity_ps *ps = trinity_get_ps(new_rps);
1495	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1496	struct trinity_power_info *pi = trinity_get_pi(rdev);
1497	u32 min_voltage = 0; /* ??? */
1498	u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1499	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1500	u32 i;
1501	u16 min_vce_voltage;
1502	bool force_high;
1503	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1504
1505	if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1506		return trinity_patch_thermal_state(rdev, ps, current_ps);
1507
1508	trinity_adjust_uvd_state(rdev, new_rps);
1509
1510	if (new_rps->vce_active) {
1511		new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1512		new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1513	} else {
1514		new_rps->evclk = 0;
1515		new_rps->ecclk = 0;
1516	}
1517
1518	for (i = 0; i < ps->num_levels; i++) {
1519		if (ps->levels[i].vddc_index < min_voltage)
1520			ps->levels[i].vddc_index = min_voltage;
1521
1522		if (ps->levels[i].sclk < min_sclk)
1523			ps->levels[i].sclk =
1524				trinity_get_valid_engine_clock(rdev, min_sclk);
1525
1526		/* patch in vce limits */
1527		if (new_rps->vce_active) {
1528			/* sclk */
1529			if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1530				ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1531			/* vddc */
1532			trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1533			if (ps->levels[i].vddc_index < min_vce_voltage)
1534				ps->levels[i].vddc_index = min_vce_voltage;
1535		}
1536
1537		ps->levels[i].ds_divider_index =
1538			sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1539
1540		ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1541
1542		ps->levels[i].allow_gnb_slow = 1;
1543		ps->levels[i].force_nbp_state = 0;
1544		ps->levels[i].display_wm =
1545			trinity_calculate_display_wm(rdev, ps, i);
1546		ps->levels[i].vce_wm =
1547			trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1548	}
1549
1550	if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1551	    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1552		ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1553
1554	if (pi->sys_info.nb_dpm_enable) {
1555		ps->Dpm0PgNbPsLo = 0x1;
1556		ps->Dpm0PgNbPsHi = 0x0;
1557		ps->DpmXNbPsLo = 0x2;
1558		ps->DpmXNbPsHi = 0x1;
1559
1560		if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1561		    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1562			force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1563				      ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1564				       (pi->sys_info.uma_channel_number == 1)));
1565			force_high = (num_active_displays >= 3) || force_high;
1566			ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1567			ps->Dpm0PgNbPsHi = 0x1;
1568			ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1569			ps->DpmXNbPsHi = 0x2;
1570			ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1571		}
1572	}
1573}
1574
1575static void trinity_cleanup_asic(struct radeon_device *rdev)
1576{
1577	sumo_take_smu_control(rdev, false);
1578}
1579
1580#if 0
1581static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1582{
1583	struct trinity_power_info *pi = trinity_get_pi(rdev);
1584
1585	if (pi->voltage_drop_in_dce)
1586		trinity_dce_enable_voltage_adjustment(rdev, false);
1587}
1588#endif
1589
1590static void trinity_add_dccac_value(struct radeon_device *rdev)
1591{
1592	u32 gpu_cac_avrg_cntl_window_size;
1593	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1594	u64 disp_clk = rdev->clock.default_dispclk / 100;
1595	u32 dc_cac_value;
1596
1597	gpu_cac_avrg_cntl_window_size =
1598		(RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1599
1600	dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1601			     (32 - gpu_cac_avrg_cntl_window_size));
1602
1603	WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1604}
1605
1606void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1607{
1608	struct trinity_power_info *pi = trinity_get_pi(rdev);
1609
1610	if (pi->voltage_drop_in_dce)
1611		trinity_dce_enable_voltage_adjustment(rdev, true);
1612	trinity_add_dccac_value(rdev);
1613}
1614
1615union power_info {
1616	struct _ATOM_POWERPLAY_INFO info;
1617	struct _ATOM_POWERPLAY_INFO_V2 info_2;
1618	struct _ATOM_POWERPLAY_INFO_V3 info_3;
1619	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1620	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1621	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1622};
1623
1624union pplib_clock_info {
1625	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1626	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1627	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1628	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1629};
1630
1631union pplib_power_state {
1632	struct _ATOM_PPLIB_STATE v1;
1633	struct _ATOM_PPLIB_STATE_V2 v2;
1634};
1635
1636static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1637					       struct radeon_ps *rps,
1638					       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1639					       u8 table_rev)
1640{
1641	struct trinity_ps *ps = trinity_get_ps(rps);
1642
1643	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1644	rps->class = le16_to_cpu(non_clock_info->usClassification);
1645	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1646
1647	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1648		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1649		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1650	} else {
1651		rps->vclk = 0;
1652		rps->dclk = 0;
1653	}
1654
1655	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1656		rdev->pm.dpm.boot_ps = rps;
1657		trinity_patch_boot_state(rdev, ps);
1658	}
1659	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1660		rdev->pm.dpm.uvd_ps = rps;
1661}
1662
1663static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1664					   struct radeon_ps *rps, int index,
1665					   union pplib_clock_info *clock_info)
1666{
1667	struct trinity_power_info *pi = trinity_get_pi(rdev);
1668	struct trinity_ps *ps = trinity_get_ps(rps);
1669	struct trinity_pl *pl = &ps->levels[index];
1670	u32 sclk;
1671
1672	sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1673	sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1674	pl->sclk = sclk;
1675	pl->vddc_index = clock_info->sumo.vddcIndex;
1676
1677	ps->num_levels = index + 1;
1678
1679	if (pi->enable_sclk_ds) {
1680		pl->ds_divider_index = 5;
1681		pl->ss_divider_index = 5;
1682	}
1683}
1684
1685static int trinity_parse_power_table(struct radeon_device *rdev)
1686{
1687	struct radeon_mode_info *mode_info = &rdev->mode_info;
1688	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1689	union pplib_power_state *power_state;
1690	int i, j, k, non_clock_array_index, clock_array_index;
1691	union pplib_clock_info *clock_info;
1692	struct _StateArray *state_array;
1693	struct _ClockInfoArray *clock_info_array;
1694	struct _NonClockInfoArray *non_clock_info_array;
1695	union power_info *power_info;
1696	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1697	u16 data_offset;
1698	u8 frev, crev;
1699	u8 *power_state_offset;
1700	struct sumo_ps *ps;
1701
1702	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1703				   &frev, &crev, &data_offset))
1704		return -EINVAL;
1705	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1706
1707	state_array = (struct _StateArray *)
1708		(mode_info->atom_context->bios + data_offset +
1709		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
1710	clock_info_array = (struct _ClockInfoArray *)
1711		(mode_info->atom_context->bios + data_offset +
1712		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1713	non_clock_info_array = (struct _NonClockInfoArray *)
1714		(mode_info->atom_context->bios + data_offset +
1715		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1716
1717	rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1718				  sizeof(struct radeon_ps),
1719				  GFP_KERNEL);
1720	if (!rdev->pm.dpm.ps)
1721		return -ENOMEM;
1722	power_state_offset = (u8 *)state_array->states;
1723	for (i = 0; i < state_array->ucNumEntries; i++) {
1724		u8 *idx;
1725		power_state = (union pplib_power_state *)power_state_offset;
1726		non_clock_array_index = power_state->v2.nonClockInfoIndex;
1727		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1728			&non_clock_info_array->nonClockInfo[non_clock_array_index];
1729		if (!rdev->pm.power_state[i].clock_info)
1730			return -EINVAL;
1731		ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1732		if (ps == NULL) {
1733			kfree(rdev->pm.dpm.ps);
1734			return -ENOMEM;
1735		}
1736		rdev->pm.dpm.ps[i].ps_priv = ps;
1737		k = 0;
1738		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1739		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1740			clock_array_index = idx[j];
1741			if (clock_array_index >= clock_info_array->ucNumEntries)
1742				continue;
1743			if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1744				break;
1745			clock_info = (union pplib_clock_info *)
1746				((u8 *)&clock_info_array->clockInfo[0] +
1747				 (clock_array_index * clock_info_array->ucEntrySize));
1748			trinity_parse_pplib_clock_info(rdev,
1749						       &rdev->pm.dpm.ps[i], k,
1750						       clock_info);
1751			k++;
1752		}
1753		trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1754						   non_clock_info,
1755						   non_clock_info_array->ucEntrySize);
1756		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1757	}
1758	rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1759
1760	/* fill in the vce power states */
1761	for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1762		u32 sclk;
1763		clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1764		clock_info = (union pplib_clock_info *)
1765			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1766		sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1767		sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1768		rdev->pm.dpm.vce_states[i].sclk = sclk;
1769		rdev->pm.dpm.vce_states[i].mclk = 0;
1770	}
1771
1772	return 0;
1773}
1774
1775union igp_info {
1776	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1777	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1778	struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1779	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1780	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1781};
1782
1783static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1784{
1785	struct trinity_power_info *pi = trinity_get_pi(rdev);
1786	u32 divider;
1787
1788	if (did >= 8 && did <= 0x3f)
1789		divider = did * 25;
1790	else if (did > 0x3f && did <= 0x5f)
1791		divider = (did - 64) * 50 + 1600;
1792	else if (did > 0x5f && did <= 0x7e)
1793		divider = (did - 96) * 100 + 3200;
1794	else if (did == 0x7f)
1795		divider = 128 * 100;
1796	else
1797		return 10000;
1798
1799	return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1800}
1801
1802static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1803{
1804	struct trinity_power_info *pi = trinity_get_pi(rdev);
1805	struct radeon_mode_info *mode_info = &rdev->mode_info;
1806	int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1807	union igp_info *igp_info;
1808	u8 frev, crev;
1809	u16 data_offset;
1810	int i;
1811
1812	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1813				   &frev, &crev, &data_offset)) {
1814		igp_info = (union igp_info *)(mode_info->atom_context->bios +
1815					      data_offset);
1816
1817		if (crev != 7) {
1818			DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1819			return -EINVAL;
1820		}
1821		pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1822		pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1823		pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1824		pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1825		pi->sys_info.bootup_nb_voltage_index =
1826			le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1827		if (igp_info->info_7.ucHtcTmpLmt == 0)
1828			pi->sys_info.htc_tmp_lmt = 203;
1829		else
1830			pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1831		if (igp_info->info_7.ucHtcHystLmt == 0)
1832			pi->sys_info.htc_hyst_lmt = 5;
1833		else
1834			pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1835		if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1836			DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1837		}
1838
1839		if (pi->enable_nbps_policy)
1840			pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1841		else
1842			pi->sys_info.nb_dpm_enable = 0;
1843
1844		for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1845			pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1846			pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1847		}
1848
1849		pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1850		pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1851		pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1852		pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1853
1854		if (!pi->sys_info.nb_dpm_enable) {
1855			for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1856				pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1857				pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1858				pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1859			}
1860		}
1861
1862		pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1863
1864		sumo_construct_sclk_voltage_mapping_table(rdev,
1865							  &pi->sys_info.sclk_voltage_mapping_table,
1866							  igp_info->info_7.sAvail_SCLK);
1867		sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1868						 igp_info->info_7.sAvail_SCLK);
1869
1870		pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1871			igp_info->info_7.ucDPMState0VclkFid;
1872		pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1873			igp_info->info_7.ucDPMState1VclkFid;
1874		pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1875			igp_info->info_7.ucDPMState2VclkFid;
1876		pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1877			igp_info->info_7.ucDPMState3VclkFid;
1878
1879		pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1880			igp_info->info_7.ucDPMState0DclkFid;
1881		pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1882			igp_info->info_7.ucDPMState1DclkFid;
1883		pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1884			igp_info->info_7.ucDPMState2DclkFid;
1885		pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1886			igp_info->info_7.ucDPMState3DclkFid;
1887
1888		for (i = 0; i < 4; i++) {
1889			pi->sys_info.uvd_clock_table_entries[i].vclk =
1890				trinity_convert_did_to_freq(rdev,
1891							    pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1892			pi->sys_info.uvd_clock_table_entries[i].dclk =
1893				trinity_convert_did_to_freq(rdev,
1894							    pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1895		}
1896
1897
1898
1899	}
1900	return 0;
1901}
1902
1903int trinity_dpm_init(struct radeon_device *rdev)
1904{
1905	struct trinity_power_info *pi;
1906	int ret, i;
1907
1908	pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1909	if (pi == NULL)
1910		return -ENOMEM;
1911	rdev->pm.dpm.priv = pi;
1912
1913	for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1914		pi->at[i] = TRINITY_AT_DFLT;
1915
1916	if (radeon_bapm == -1) {
1917		/* There are stability issues reported on with
1918		 * bapm enabled when switching between AC and battery
1919		 * power.  At the same time, some MSI boards hang
1920		 * if it's not enabled and dpm is enabled.  Just enable
1921		 * it for MSI boards right now.
1922		 */
1923		if (rdev->pdev->subsystem_vendor == 0x1462)
1924			pi->enable_bapm = true;
1925		else
1926			pi->enable_bapm = false;
1927	} else if (radeon_bapm == 0) {
1928		pi->enable_bapm = false;
1929	} else {
1930		pi->enable_bapm = true;
1931	}
1932	pi->enable_nbps_policy = true;
1933	pi->enable_sclk_ds = true;
1934	pi->enable_gfx_power_gating = true;
1935	pi->enable_gfx_clock_gating = true;
1936	pi->enable_mg_clock_gating = false;
1937	pi->enable_gfx_dynamic_mgpg = false;
1938	pi->override_dynamic_mgpg = false;
1939	pi->enable_auto_thermal_throttling = true;
1940	pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1941	pi->uvd_dpm = true; /* ??? */
1942
1943	ret = trinity_parse_sys_info_table(rdev);
1944	if (ret)
1945		return ret;
1946
1947	trinity_construct_boot_state(rdev);
1948
1949	ret = r600_get_platform_caps(rdev);
1950	if (ret)
1951		return ret;
1952
1953	ret = r600_parse_extended_power_table(rdev);
1954	if (ret)
1955		return ret;
1956
1957	ret = trinity_parse_power_table(rdev);
1958	if (ret)
1959		return ret;
1960
1961	pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
1962	pi->enable_dpm = true;
1963
1964	return 0;
1965}
1966
1967void trinity_dpm_print_power_state(struct radeon_device *rdev,
1968				   struct radeon_ps *rps)
1969{
1970	int i;
1971	struct trinity_ps *ps = trinity_get_ps(rps);
1972
1973	r600_dpm_print_class_info(rps->class, rps->class2);
1974	r600_dpm_print_cap_info(rps->caps);
1975	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1976	for (i = 0; i < ps->num_levels; i++) {
1977		struct trinity_pl *pl = &ps->levels[i];
1978		printk("\t\tpower level %d    sclk: %u vddc: %u\n",
1979		       i, pl->sclk,
1980		       trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
1981	}
1982	r600_dpm_print_ps_status(rdev, rps);
1983}
1984
1985void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
1986							 struct seq_file *m)
1987{
1988	struct trinity_power_info *pi = trinity_get_pi(rdev);
1989	struct radeon_ps *rps = &pi->current_rps;
1990	struct trinity_ps *ps = trinity_get_ps(rps);
1991	struct trinity_pl *pl;
1992	u32 current_index =
1993		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
1994		CURRENT_STATE_SHIFT;
1995
1996	if (current_index >= ps->num_levels) {
1997		seq_printf(m, "invalid dpm profile %d\n", current_index);
1998	} else {
1999		pl = &ps->levels[current_index];
2000		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2001		seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2002			   current_index, pl->sclk,
2003			   trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2004	}
2005}
2006
2007u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2008{
2009	struct trinity_power_info *pi = trinity_get_pi(rdev);
2010	struct radeon_ps *rps = &pi->current_rps;
2011	struct trinity_ps *ps = trinity_get_ps(rps);
2012	struct trinity_pl *pl;
2013	u32 current_index =
2014		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2015		CURRENT_STATE_SHIFT;
2016
2017	if (current_index >= ps->num_levels) {
2018		return 0;
2019	} else {
2020		pl = &ps->levels[current_index];
2021		return pl->sclk;
2022	}
2023}
2024
2025u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2026{
2027	struct trinity_power_info *pi = trinity_get_pi(rdev);
2028
2029	return pi->sys_info.bootup_uma_clk;
2030}
2031
2032void trinity_dpm_fini(struct radeon_device *rdev)
2033{
2034	int i;
2035
2036	trinity_cleanup_asic(rdev); /* ??? */
2037
2038	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2039		kfree(rdev->pm.dpm.ps[i].ps_priv);
2040	}
2041	kfree(rdev->pm.dpm.ps);
2042	kfree(rdev->pm.dpm.priv);
2043	r600_free_extended_power_table(rdev);
2044}
2045
2046u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2047{
2048	struct trinity_power_info *pi = trinity_get_pi(rdev);
2049	struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2050
2051	if (low)
2052		return requested_state->levels[0].sclk;
2053	else
2054		return requested_state->levels[requested_state->num_levels - 1].sclk;
2055}
2056
2057u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2058{
2059	struct trinity_power_info *pi = trinity_get_pi(rdev);
2060
2061	return pi->sys_info.bootup_uma_clk;
2062}