1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/pci.h>
  25#include <linux/seq_file.h>
  26
  27#include "r600_dpm.h"
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "trinity_dpm.h"
  31#include "trinityd.h"
  32#include "vce.h"
 
 
  33
  34#define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
  35#define TRINITY_MINIMUM_ENGINE_CLOCK 800
  36#define SCLK_MIN_DIV_INTV_SHIFT     12
  37#define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
  38
  39#ifndef TRINITY_MGCG_SEQUENCE
  40#define TRINITY_MGCG_SEQUENCE  100
  41
  42static const u32 trinity_mgcg_shls_default[] = {
 
  43	/* Register, Value, Mask */
  44	0x0000802c, 0xc0000000, 0xffffffff,
  45	0x00003fc4, 0xc0000000, 0xffffffff,
  46	0x00005448, 0x00000100, 0xffffffff,
  47	0x000055e4, 0x00000100, 0xffffffff,
  48	0x0000160c, 0x00000100, 0xffffffff,
  49	0x00008984, 0x06000100, 0xffffffff,
  50	0x0000c164, 0x00000100, 0xffffffff,
  51	0x00008a18, 0x00000100, 0xffffffff,
  52	0x0000897c, 0x06000100, 0xffffffff,
  53	0x00008b28, 0x00000100, 0xffffffff,
  54	0x00009144, 0x00800200, 0xffffffff,
  55	0x00009a60, 0x00000100, 0xffffffff,
  56	0x00009868, 0x00000100, 0xffffffff,
  57	0x00008d58, 0x00000100, 0xffffffff,
  58	0x00009510, 0x00000100, 0xffffffff,
  59	0x0000949c, 0x00000100, 0xffffffff,
  60	0x00009654, 0x00000100, 0xffffffff,
  61	0x00009030, 0x00000100, 0xffffffff,
  62	0x00009034, 0x00000100, 0xffffffff,
  63	0x00009038, 0x00000100, 0xffffffff,
  64	0x0000903c, 0x00000100, 0xffffffff,
  65	0x00009040, 0x00000100, 0xffffffff,
  66	0x0000a200, 0x00000100, 0xffffffff,
  67	0x0000a204, 0x00000100, 0xffffffff,
  68	0x0000a208, 0x00000100, 0xffffffff,
  69	0x0000a20c, 0x00000100, 0xffffffff,
  70	0x00009744, 0x00000100, 0xffffffff,
  71	0x00003f80, 0x00000100, 0xffffffff,
  72	0x0000a210, 0x00000100, 0xffffffff,
  73	0x0000a214, 0x00000100, 0xffffffff,
  74	0x000004d8, 0x00000100, 0xffffffff,
  75	0x00009664, 0x00000100, 0xffffffff,
  76	0x00009698, 0x00000100, 0xffffffff,
  77	0x000004d4, 0x00000200, 0xffffffff,
  78	0x000004d0, 0x00000000, 0xffffffff,
  79	0x000030cc, 0x00000104, 0xffffffff,
  80	0x0000d0c0, 0x00000100, 0xffffffff,
  81	0x0000d8c0, 0x00000100, 0xffffffff,
  82	0x0000951c, 0x00010000, 0xffffffff,
  83	0x00009160, 0x00030002, 0xffffffff,
  84	0x00009164, 0x00050004, 0xffffffff,
  85	0x00009168, 0x00070006, 0xffffffff,
  86	0x00009178, 0x00070000, 0xffffffff,
  87	0x0000917c, 0x00030002, 0xffffffff,
  88	0x00009180, 0x00050004, 0xffffffff,
  89	0x0000918c, 0x00010006, 0xffffffff,
  90	0x00009190, 0x00090008, 0xffffffff,
  91	0x00009194, 0x00070000, 0xffffffff,
  92	0x00009198, 0x00030002, 0xffffffff,
  93	0x0000919c, 0x00050004, 0xffffffff,
  94	0x000091a8, 0x00010006, 0xffffffff,
  95	0x000091ac, 0x00090008, 0xffffffff,
  96	0x000091b0, 0x00070000, 0xffffffff,
  97	0x000091b4, 0x00030002, 0xffffffff,
  98	0x000091b8, 0x00050004, 0xffffffff,
  99	0x000091c4, 0x00010006, 0xffffffff,
 100	0x000091c8, 0x00090008, 0xffffffff,
 101	0x000091cc, 0x00070000, 0xffffffff,
 102	0x000091d0, 0x00030002, 0xffffffff,
 103	0x000091d4, 0x00050004, 0xffffffff,
 104	0x000091e0, 0x00010006, 0xffffffff,
 105	0x000091e4, 0x00090008, 0xffffffff,
 106	0x000091e8, 0x00000000, 0xffffffff,
 107	0x000091ec, 0x00070000, 0xffffffff,
 108	0x000091f0, 0x00030002, 0xffffffff,
 109	0x000091f4, 0x00050004, 0xffffffff,
 110	0x00009200, 0x00010006, 0xffffffff,
 111	0x00009204, 0x00090008, 0xffffffff,
 112	0x00009208, 0x00070000, 0xffffffff,
 113	0x0000920c, 0x00030002, 0xffffffff,
 114	0x00009210, 0x00050004, 0xffffffff,
 115	0x0000921c, 0x00010006, 0xffffffff,
 116	0x00009220, 0x00090008, 0xffffffff,
 117	0x00009294, 0x00000000, 0xffffffff
 118};
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 119#endif
 120
 121#ifndef TRINITY_SYSLS_SEQUENCE
 122#define TRINITY_SYSLS_SEQUENCE  100
 123
 124static const u32 trinity_sysls_disable[] = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 125	/* Register, Value, Mask */
 126	0x0000d0c0, 0x00000000, 0xffffffff,
 127	0x0000d8c0, 0x00000000, 0xffffffff,
 128	0x000055e8, 0x00000000, 0xffffffff,
 129	0x0000d0bc, 0x00000000, 0xffffffff,
 130	0x0000d8bc, 0x00000000, 0xffffffff,
 131	0x000015c0, 0x00041401, 0xffffffff,
 132	0x0000264c, 0x00040400, 0xffffffff,
 133	0x00002648, 0x00040400, 0xffffffff,
 134	0x00002650, 0x00040400, 0xffffffff,
 135	0x000020b8, 0x00040400, 0xffffffff,
 136	0x000020bc, 0x00040400, 0xffffffff,
 137	0x000020c0, 0x00040c80, 0xffffffff,
 138	0x0000f4a0, 0x000000c0, 0xffffffff,
 139	0x0000f4a4, 0x00680000, 0xffffffff,
 140	0x00002f50, 0x00000404, 0xffffffff,
 141	0x000004c8, 0x00000001, 0xffffffff,
 142	0x0000641c, 0x00007ffd, 0xffffffff,
 143	0x00000c7c, 0x0000ff00, 0xffffffff,
 144	0x00006dfc, 0x0000007f, 0xffffffff
 145};
 146
 147static const u32 trinity_sysls_enable[] = {
 
 148	/* Register, Value, Mask */
 149	0x000055e8, 0x00000001, 0xffffffff,
 150	0x0000d0bc, 0x00000100, 0xffffffff,
 151	0x0000d8bc, 0x00000100, 0xffffffff,
 152	0x000015c0, 0x000c1401, 0xffffffff,
 153	0x0000264c, 0x000c0400, 0xffffffff,
 154	0x00002648, 0x000c0400, 0xffffffff,
 155	0x00002650, 0x000c0400, 0xffffffff,
 156	0x000020b8, 0x000c0400, 0xffffffff,
 157	0x000020bc, 0x000c0400, 0xffffffff,
 158	0x000020c0, 0x000c0c80, 0xffffffff,
 159	0x0000f4a0, 0x000000c0, 0xffffffff,
 160	0x0000f4a4, 0x00680fff, 0xffffffff,
 161	0x00002f50, 0x00000903, 0xffffffff,
 162	0x000004c8, 0x00000000, 0xffffffff,
 163	0x0000641c, 0x00000000, 0xffffffff,
 164	0x00000c7c, 0x00000000, 0xffffffff,
 165	0x00006dfc, 0x00000000, 0xffffffff
 166};
 167#endif
 168
 169static const u32 trinity_override_mgpg_sequences[] = {
 
 170	/* Register, Value */
 171	0x00000200, 0xE030032C,
 172	0x00000204, 0x00000FFF,
 173	0x00000200, 0xE0300058,
 174	0x00000204, 0x00030301,
 175	0x00000200, 0xE0300054,
 176	0x00000204, 0x500010FF,
 177	0x00000200, 0xE0300074,
 178	0x00000204, 0x00030301,
 179	0x00000200, 0xE0300070,
 180	0x00000204, 0x500010FF,
 181	0x00000200, 0xE0300090,
 182	0x00000204, 0x00030301,
 183	0x00000200, 0xE030008C,
 184	0x00000204, 0x500010FF,
 185	0x00000200, 0xE03000AC,
 186	0x00000204, 0x00030301,
 187	0x00000200, 0xE03000A8,
 188	0x00000204, 0x500010FF,
 189	0x00000200, 0xE03000C8,
 190	0x00000204, 0x00030301,
 191	0x00000200, 0xE03000C4,
 192	0x00000204, 0x500010FF,
 193	0x00000200, 0xE03000E4,
 194	0x00000204, 0x00030301,
 195	0x00000200, 0xE03000E0,
 196	0x00000204, 0x500010FF,
 197	0x00000200, 0xE0300100,
 198	0x00000204, 0x00030301,
 199	0x00000200, 0xE03000FC,
 200	0x00000204, 0x500010FF,
 201	0x00000200, 0xE0300058,
 202	0x00000204, 0x00030303,
 203	0x00000200, 0xE0300054,
 204	0x00000204, 0x600010FF,
 205	0x00000200, 0xE0300074,
 206	0x00000204, 0x00030303,
 207	0x00000200, 0xE0300070,
 208	0x00000204, 0x600010FF,
 209	0x00000200, 0xE0300090,
 210	0x00000204, 0x00030303,
 211	0x00000200, 0xE030008C,
 212	0x00000204, 0x600010FF,
 213	0x00000200, 0xE03000AC,
 214	0x00000204, 0x00030303,
 215	0x00000200, 0xE03000A8,
 216	0x00000204, 0x600010FF,
 217	0x00000200, 0xE03000C8,
 218	0x00000204, 0x00030303,
 219	0x00000200, 0xE03000C4,
 220	0x00000204, 0x600010FF,
 221	0x00000200, 0xE03000E4,
 222	0x00000204, 0x00030303,
 223	0x00000200, 0xE03000E0,
 224	0x00000204, 0x600010FF,
 225	0x00000200, 0xE0300100,
 226	0x00000204, 0x00030303,
 227	0x00000200, 0xE03000FC,
 228	0x00000204, 0x600010FF,
 229	0x00000200, 0xE0300058,
 230	0x00000204, 0x00030303,
 231	0x00000200, 0xE0300054,
 232	0x00000204, 0x700010FF,
 233	0x00000200, 0xE0300074,
 234	0x00000204, 0x00030303,
 235	0x00000200, 0xE0300070,
 236	0x00000204, 0x700010FF,
 237	0x00000200, 0xE0300090,
 238	0x00000204, 0x00030303,
 239	0x00000200, 0xE030008C,
 240	0x00000204, 0x700010FF,
 241	0x00000200, 0xE03000AC,
 242	0x00000204, 0x00030303,
 243	0x00000200, 0xE03000A8,
 244	0x00000204, 0x700010FF,
 245	0x00000200, 0xE03000C8,
 246	0x00000204, 0x00030303,
 247	0x00000200, 0xE03000C4,
 248	0x00000204, 0x700010FF,
 249	0x00000200, 0xE03000E4,
 250	0x00000204, 0x00030303,
 251	0x00000200, 0xE03000E0,
 252	0x00000204, 0x700010FF,
 253	0x00000200, 0xE0300100,
 254	0x00000204, 0x00030303,
 255	0x00000200, 0xE03000FC,
 256	0x00000204, 0x700010FF,
 257	0x00000200, 0xE0300058,
 258	0x00000204, 0x00010303,
 259	0x00000200, 0xE0300054,
 260	0x00000204, 0x800010FF,
 261	0x00000200, 0xE0300074,
 262	0x00000204, 0x00010303,
 263	0x00000200, 0xE0300070,
 264	0x00000204, 0x800010FF,
 265	0x00000200, 0xE0300090,
 266	0x00000204, 0x00010303,
 267	0x00000200, 0xE030008C,
 268	0x00000204, 0x800010FF,
 269	0x00000200, 0xE03000AC,
 270	0x00000204, 0x00010303,
 271	0x00000200, 0xE03000A8,
 272	0x00000204, 0x800010FF,
 273	0x00000200, 0xE03000C4,
 274	0x00000204, 0x800010FF,
 275	0x00000200, 0xE03000C8,
 276	0x00000204, 0x00010303,
 277	0x00000200, 0xE03000E4,
 278	0x00000204, 0x00010303,
 279	0x00000200, 0xE03000E0,
 280	0x00000204, 0x800010FF,
 281	0x00000200, 0xE0300100,
 282	0x00000204, 0x00010303,
 283	0x00000200, 0xE03000FC,
 284	0x00000204, 0x800010FF,
 285	0x00000200, 0x0001f198,
 286	0x00000204, 0x0003ffff,
 287	0x00000200, 0x0001f19C,
 288	0x00000204, 0x3fffffff,
 289	0x00000200, 0xE030032C,
 290	0x00000204, 0x00000000,
 291};
 292
 293static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 294						   const u32 *seq, u32 count);
 295static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
 296static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
 297					     struct radeon_ps *new_rps,
 298					     struct radeon_ps *old_rps);
 299
 300static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
 301{
 302	struct trinity_ps *ps = rps->ps_priv;
 303
 304	return ps;
 305}
 306
 307static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
 308{
 309	struct trinity_power_info *pi = rdev->pm.dpm.priv;
 310
 311	return pi;
 312}
 313
 314static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
 315{
 316	struct trinity_power_info *pi = trinity_get_pi(rdev);
 317	u32 p, u;
 318	u32 value;
 319	struct atom_clock_dividers dividers;
 320	u32 xclk = radeon_get_xclk(rdev);
 321	u32 sssd = 1;
 322	int ret;
 323	u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
 324
 325	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 326					     25000, false, &dividers);
 327	if (ret)
 328		return;
 329
 330	value = RREG32_SMC(GFX_POWER_GATING_CNTL);
 331	value &= ~(SSSD_MASK | PDS_DIV_MASK);
 332	if (sssd)
 333		value |= SSSD(1);
 334	value |= PDS_DIV(dividers.post_div);
 335	WREG32_SMC(GFX_POWER_GATING_CNTL, value);
 336
 337	r600_calculate_u_and_p(500, xclk, 16, &p, &u);
 338
 339	WREG32(CG_PG_CTRL, SP(p) | SU(u));
 340
 341	WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
 342
 343	/* XXX double check hw_rev */
 344	if (pi->override_dynamic_mgpg && (hw_rev == 0))
 345		trinity_override_dynamic_mg_powergating(rdev);
 346
 347}
 348
 349#define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
 350#define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
 351#define CGTS_SM_CTRL_REG_DISABLE    0x00600000
 352#define CGTS_SM_CTRL_REG_ENABLE     0x96944200
 353
 354static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
 355					  bool enable)
 356{
 357	u32 local0;
 358	u32 local1;
 359
 360	if (enable) {
 361		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 362		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 363
 364		WREG32_CG(CG_CGTT_LOCAL_0,
 365			  (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK));
 366		WREG32_CG(CG_CGTT_LOCAL_1,
 367			  (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK));
 368
 369		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
 370	} else {
 371		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
 372
 373		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 374		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 375
 376		WREG32_CG(CG_CGTT_LOCAL_0,
 377			  CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK));
 378		WREG32_CG(CG_CGTT_LOCAL_1,
 379			  CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK));
 380	}
 381}
 382
 383static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
 384{
 385	u32 count;
 386	const u32 *seq = NULL;
 387
 388	seq = &trinity_mgcg_shls_default[0];
 389	count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
 390
 391	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 392}
 393
 394static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
 395					   bool enable)
 396{
 397	if (enable) {
 398		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 399	} else {
 400		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 401		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 402		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 403		RREG32(GB_ADDR_CONFIG);
 404	}
 405}
 406
 407static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 408						   const u32 *seq, u32 count)
 409{
 410	u32 i, length = count * 3;
 411
 412	for (i = 0; i < length; i += 3)
 413		WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
 414}
 415
 416static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
 417						    const u32 *seq, u32 count)
 418{
 419	u32  i, length = count * 2;
 420
 421	for (i = 0; i < length; i += 2)
 422		WREG32(seq[i], seq[i+1]);
 423
 424}
 425
 426static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
 427{
 428	u32 count;
 429	const u32 *seq = NULL;
 430
 431	seq = &trinity_override_mgpg_sequences[0];
 432	count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
 433
 434	trinity_program_override_mgpg_sequences(rdev, seq, count);
 435}
 436
 437static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
 438					  bool enable)
 439{
 440	u32 count;
 441	const u32 *seq = NULL;
 442
 443	if (enable) {
 444		seq = &trinity_sysls_enable[0];
 445		count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
 446	} else {
 447		seq = &trinity_sysls_disable[0];
 448		count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
 449	}
 450
 451	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 452}
 453
 454static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
 455					   bool enable)
 456{
 457	if (enable) {
 458		if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
 459			WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
 460
 461		WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
 462	} else {
 463		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
 464		RREG32(GB_ADDR_CONFIG);
 465	}
 466}
 467
 468static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
 469					    bool enable)
 470{
 471	u32 value;
 472
 473	if (enable) {
 474		value = RREG32_SMC(PM_I_CNTL_1);
 475		value &= ~DS_PG_CNTL_MASK;
 476		value |= DS_PG_CNTL(1);
 477		WREG32_SMC(PM_I_CNTL_1, value);
 478
 479		value = RREG32_SMC(SMU_S_PG_CNTL);
 480		value &= ~DS_PG_EN_MASK;
 481		value |= DS_PG_EN(1);
 482		WREG32_SMC(SMU_S_PG_CNTL, value);
 483	} else {
 484		value = RREG32_SMC(SMU_S_PG_CNTL);
 485		value &= ~DS_PG_EN_MASK;
 486		WREG32_SMC(SMU_S_PG_CNTL, value);
 487
 488		value = RREG32_SMC(PM_I_CNTL_1);
 489		value &= ~DS_PG_CNTL_MASK;
 490		WREG32_SMC(PM_I_CNTL_1, value);
 491	}
 492
 493	trinity_gfx_dynamic_mgpg_config(rdev);
 494
 495}
 496
 497static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
 498{
 499	struct trinity_power_info *pi = trinity_get_pi(rdev);
 500
 501	if (pi->enable_gfx_clock_gating)
 502		sumo_gfx_clockgating_initialize(rdev);
 503	if (pi->enable_mg_clock_gating)
 504		trinity_mg_clockgating_initialize(rdev);
 505	if (pi->enable_gfx_power_gating)
 506		trinity_gfx_powergating_initialize(rdev);
 507	if (pi->enable_mg_clock_gating) {
 508		trinity_ls_clockgating_enable(rdev, true);
 509		trinity_mg_clockgating_enable(rdev, true);
 510	}
 511	if (pi->enable_gfx_clock_gating)
 512		trinity_gfx_clockgating_enable(rdev, true);
 513	if (pi->enable_gfx_dynamic_mgpg)
 514		trinity_gfx_dynamic_mgpg_enable(rdev, true);
 515	if (pi->enable_gfx_power_gating)
 516		trinity_gfx_powergating_enable(rdev, true);
 517}
 518
 519static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
 520{
 521	struct trinity_power_info *pi = trinity_get_pi(rdev);
 522
 523	if (pi->enable_gfx_power_gating)
 524		trinity_gfx_powergating_enable(rdev, false);
 525	if (pi->enable_gfx_dynamic_mgpg)
 526		trinity_gfx_dynamic_mgpg_enable(rdev, false);
 527	if (pi->enable_gfx_clock_gating)
 528		trinity_gfx_clockgating_enable(rdev, false);
 529	if (pi->enable_mg_clock_gating) {
 530		trinity_mg_clockgating_enable(rdev, false);
 531		trinity_ls_clockgating_enable(rdev, false);
 532	}
 533}
 534
 535static void trinity_set_divider_value(struct radeon_device *rdev,
 536				      u32 index, u32 sclk)
 537{
 538	struct atom_clock_dividers  dividers;
 539	int ret;
 540	u32 value;
 541	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 542
 543	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 544					     sclk, false, &dividers);
 545	if (ret)
 546		return;
 547
 548	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 549	value &= ~CLK_DIVIDER_MASK;
 550	value |= CLK_DIVIDER(dividers.post_div);
 551	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 552
 553	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 554					     sclk/2, false, &dividers);
 555	if (ret)
 556		return;
 557
 558	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
 559	value &= ~PD_SCLK_DIVIDER_MASK;
 560	value |= PD_SCLK_DIVIDER(dividers.post_div);
 561	WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
 562}
 563
 564static void trinity_set_ds_dividers(struct radeon_device *rdev,
 565				    u32 index, u32 divider)
 566{
 567	u32 value;
 568	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 569
 570	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 571	value &= ~DS_DIV_MASK;
 572	value |= DS_DIV(divider);
 573	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 574}
 575
 576static void trinity_set_ss_dividers(struct radeon_device *rdev,
 577				    u32 index, u32 divider)
 578{
 579	u32 value;
 580	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 581
 582	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 583	value &= ~DS_SH_DIV_MASK;
 584	value |= DS_SH_DIV(divider);
 585	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 586}
 587
 588static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
 589{
 590	struct trinity_power_info *pi = trinity_get_pi(rdev);
 591	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
 592	u32 value;
 593	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 594
 595	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 596	value &= ~VID_MASK;
 597	value |= VID(vid_7bit);
 598	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 599
 600	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 601	value &= ~LVRT_MASK;
 602	value |= LVRT(0);
 603	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 604}
 605
 606static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
 607				       u32 index, u32 gnb_slow)
 608{
 609	u32 value;
 610	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 611
 612	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 613	value &= ~GNB_SLOW_MASK;
 614	value |= GNB_SLOW(gnb_slow);
 615	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 616}
 617
 618static void trinity_set_force_nbp_state(struct radeon_device *rdev,
 619					u32 index, u32 force_nbp_state)
 620{
 621	u32 value;
 622	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 623
 624	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 625	value &= ~FORCE_NBPS1_MASK;
 626	value |= FORCE_NBPS1(force_nbp_state);
 627	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 628}
 629
 630static void trinity_set_display_wm(struct radeon_device *rdev,
 631				   u32 index, u32 wm)
 632{
 633	u32 value;
 634	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 635
 636	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 637	value &= ~DISPLAY_WM_MASK;
 638	value |= DISPLAY_WM(wm);
 639	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 640}
 641
 642static void trinity_set_vce_wm(struct radeon_device *rdev,
 643			       u32 index, u32 wm)
 644{
 645	u32 value;
 646	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 647
 648	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 649	value &= ~VCE_WM_MASK;
 650	value |= VCE_WM(wm);
 651	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 652}
 653
 654static void trinity_set_at(struct radeon_device *rdev,
 655			   u32 index, u32 at)
 656{
 657	u32 value;
 658	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 659
 660	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
 661	value &= ~AT_MASK;
 662	value |= AT(at);
 663	WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
 664}
 665
 666static void trinity_program_power_level(struct radeon_device *rdev,
 667					struct trinity_pl *pl, u32 index)
 668{
 669	struct trinity_power_info *pi = trinity_get_pi(rdev);
 670
 671	if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
 672		return;
 673
 674	trinity_set_divider_value(rdev, index, pl->sclk);
 675	trinity_set_vid(rdev, index, pl->vddc_index);
 676	trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
 677	trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
 678	trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
 679	trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
 680	trinity_set_display_wm(rdev, index, pl->display_wm);
 681	trinity_set_vce_wm(rdev, index, pl->vce_wm);
 682	trinity_set_at(rdev, index, pi->at[index]);
 683}
 684
 685static void trinity_power_level_enable_disable(struct radeon_device *rdev,
 686					       u32 index, bool enable)
 687{
 688	u32 value;
 689	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 690
 691	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 692	value &= ~STATE_VALID_MASK;
 693	if (enable)
 694		value |= STATE_VALID(1);
 695	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 696}
 697
 698static bool trinity_dpm_enabled(struct radeon_device *rdev)
 699{
 700	if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
 701		return true;
 702	else
 703		return false;
 704}
 705
 706static void trinity_start_dpm(struct radeon_device *rdev)
 707{
 708	u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 709
 710	value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
 711	value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
 712	WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
 713
 714	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 715	WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
 716
 717	trinity_dpm_config(rdev, true);
 718}
 719
 720static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
 721{
 722	int i;
 723
 724	for (i = 0; i < rdev->usec_timeout; i++) {
 725		if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
 726			break;
 727		udelay(1);
 728	}
 729	for (i = 0; i < rdev->usec_timeout; i++) {
 730		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
 731			break;
 732		udelay(1);
 733	}
 734	for (i = 0; i < rdev->usec_timeout; i++) {
 735		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 736			break;
 737		udelay(1);
 738	}
 739}
 740
 741static void trinity_stop_dpm(struct radeon_device *rdev)
 742{
 743	u32 sclk_dpm_cntl;
 744
 745	WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
 746
 747	sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 748	sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
 749	WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
 750
 751	trinity_dpm_config(rdev, false);
 752}
 753
 754static void trinity_start_am(struct radeon_device *rdev)
 755{
 756	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 757}
 758
 759static void trinity_reset_am(struct radeon_device *rdev)
 760{
 761	WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
 762		 ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 763}
 764
 765static void trinity_wait_for_level_0(struct radeon_device *rdev)
 766{
 767	int i;
 768
 769	for (i = 0; i < rdev->usec_timeout; i++) {
 770		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 771			break;
 772		udelay(1);
 773	}
 774}
 775
 776static void trinity_enable_power_level_0(struct radeon_device *rdev)
 777{
 778	trinity_power_level_enable_disable(rdev, 0, true);
 779}
 780
 781static void trinity_force_level_0(struct radeon_device *rdev)
 782{
 783	trinity_dpm_force_state(rdev, 0);
 784}
 785
 786static void trinity_unforce_levels(struct radeon_device *rdev)
 787{
 788	trinity_dpm_no_forced_level(rdev);
 789}
 790
 791static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
 792						struct radeon_ps *new_rps,
 793						struct radeon_ps *old_rps)
 794{
 795	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 796	struct trinity_ps *old_ps = trinity_get_ps(old_rps);
 797	u32 i;
 798	u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
 799
 800	for (i = 0; i < new_ps->num_levels; i++) {
 801		trinity_program_power_level(rdev, &new_ps->levels[i], i);
 802		trinity_power_level_enable_disable(rdev, i, true);
 803	}
 804
 805	for (i = new_ps->num_levels; i < n_current_state_levels; i++)
 806		trinity_power_level_enable_disable(rdev, i, false);
 807}
 808
 809static void trinity_program_bootup_state(struct radeon_device *rdev)
 810{
 811	struct trinity_power_info *pi = trinity_get_pi(rdev);
 812	u32 i;
 813
 814	trinity_program_power_level(rdev, &pi->boot_pl, 0);
 815	trinity_power_level_enable_disable(rdev, 0, true);
 816
 817	for (i = 1; i < 8; i++)
 818		trinity_power_level_enable_disable(rdev, i, false);
 819}
 820
 821static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
 822					  struct radeon_ps *rps)
 823{
 824	struct trinity_ps *ps = trinity_get_ps(rps);
 825	u32 uvdstates = (ps->vclk_low_divider |
 826			 ps->vclk_high_divider << 8 |
 827			 ps->dclk_low_divider << 16 |
 828			 ps->dclk_high_divider << 24);
 829
 830	WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
 831}
 832
 833static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
 834					   u32 interval)
 835{
 836	u32 p, u;
 837	u32 tp = RREG32_SMC(PM_TP);
 838	u32 val;
 839	u32 xclk = radeon_get_xclk(rdev);
 840
 841	r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
 842
 843	val = (p + tp - 1) / tp;
 844
 845	WREG32_SMC(SMU_UVD_DPM_CNTL, val);
 846}
 847
 848static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
 849{
 850	if ((rps->vclk == 0) && (rps->dclk == 0))
 851		return true;
 852	else
 853		return false;
 854}
 855
 856static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
 857				     struct radeon_ps *rps2)
 858{
 859	struct trinity_ps *ps1 = trinity_get_ps(rps1);
 860	struct trinity_ps *ps2 = trinity_get_ps(rps2);
 861
 862	if ((rps1->vclk == rps2->vclk) &&
 863	    (rps1->dclk == rps2->dclk) &&
 864	    (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
 865	    (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
 866	    (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
 867	    (ps1->dclk_high_divider == ps2->dclk_high_divider))
 868		return true;
 869	else
 870		return false;
 871}
 872
 873static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
 874				     struct radeon_ps *new_rps,
 875				     struct radeon_ps *old_rps)
 876{
 877	struct trinity_power_info *pi = trinity_get_pi(rdev);
 878
 879	if (pi->enable_gfx_power_gating) {
 880		trinity_gfx_powergating_enable(rdev, false);
 881	}
 882
 883	if (pi->uvd_dpm) {
 884		if (trinity_uvd_clocks_zero(new_rps) &&
 885		    !trinity_uvd_clocks_zero(old_rps)) {
 886			trinity_setup_uvd_dpm_interval(rdev, 0);
 887		} else if (!trinity_uvd_clocks_zero(new_rps)) {
 888			trinity_setup_uvd_clock_table(rdev, new_rps);
 889
 890			if (trinity_uvd_clocks_zero(old_rps)) {
 891				u32 tmp = RREG32(CG_MISC_REG);
 892				tmp &= 0xfffffffd;
 893				WREG32(CG_MISC_REG, tmp);
 894
 895				radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 896
 897				trinity_setup_uvd_dpm_interval(rdev, 3000);
 898			}
 899		}
 900		trinity_uvd_dpm_config(rdev);
 901	} else {
 902		if (trinity_uvd_clocks_zero(new_rps) ||
 903		    trinity_uvd_clocks_equal(new_rps, old_rps))
 904			return;
 905
 906		radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 907	}
 908
 909	if (pi->enable_gfx_power_gating) {
 910		trinity_gfx_powergating_enable(rdev, true);
 911	}
 912}
 913
 914static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
 915						       struct radeon_ps *new_rps,
 916						       struct radeon_ps *old_rps)
 917{
 918	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 919	struct trinity_ps *current_ps = trinity_get_ps(new_rps);
 920
 921	if (new_ps->levels[new_ps->num_levels - 1].sclk >=
 922	    current_ps->levels[current_ps->num_levels - 1].sclk)
 923		return;
 924
 925	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 926}
 927
 928static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
 929						      struct radeon_ps *new_rps,
 930						      struct radeon_ps *old_rps)
 931{
 932	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 933	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
 934
 935	if (new_ps->levels[new_ps->num_levels - 1].sclk <
 936	    current_ps->levels[current_ps->num_levels - 1].sclk)
 937		return;
 938
 939	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 940}
 941
 942static void trinity_set_vce_clock(struct radeon_device *rdev,
 943				  struct radeon_ps *new_rps,
 944				  struct radeon_ps *old_rps)
 945{
 946	if ((old_rps->evclk != new_rps->evclk) ||
 947	    (old_rps->ecclk != new_rps->ecclk)) {
 948		/* turn the clocks on when encoding, off otherwise */
 949		if (new_rps->evclk || new_rps->ecclk)
 950			vce_v1_0_enable_mgcg(rdev, false);
 951		else
 952			vce_v1_0_enable_mgcg(rdev, true);
 953		radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
 954	}
 955}
 956
 957static void trinity_program_ttt(struct radeon_device *rdev)
 958{
 959	struct trinity_power_info *pi = trinity_get_pi(rdev);
 960	u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
 961
 962	value &= ~(HT_MASK | LT_MASK);
 963	value |= HT((pi->thermal_auto_throttling + 49) * 8);
 964	value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
 965	WREG32_SMC(SMU_SCLK_DPM_TTT, value);
 966}
 967
 968static void trinity_enable_att(struct radeon_device *rdev)
 969{
 970	u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
 971
 972	value &= ~SCLK_TT_EN_MASK;
 973	value |= SCLK_TT_EN(1);
 974	WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
 975}
 976
 977static void trinity_program_sclk_dpm(struct radeon_device *rdev)
 978{
 979	u32 p, u;
 980	u32 tp = RREG32_SMC(PM_TP);
 981	u32 ni;
 982	u32 xclk = radeon_get_xclk(rdev);
 983	u32 value;
 984
 985	r600_calculate_u_and_p(400, xclk, 16, &p, &u);
 986
 987	ni = (p + tp - 1) / tp;
 988
 989	value = RREG32_SMC(PM_I_CNTL_1);
 990	value &= ~SCLK_DPM_MASK;
 991	value |= SCLK_DPM(ni);
 992	WREG32_SMC(PM_I_CNTL_1, value);
 993}
 994
 995static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
 996						 int min_temp, int max_temp)
 997{
 998	int low_temp = 0 * 1000;
 999	int high_temp = 255 * 1000;
1000
1001	if (low_temp < min_temp)
1002		low_temp = min_temp;
1003	if (high_temp > max_temp)
1004		high_temp = max_temp;
1005	if (high_temp < low_temp) {
1006		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1007		return -EINVAL;
1008	}
1009
1010	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1011	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1012
1013	rdev->pm.dpm.thermal.min_temp = low_temp;
1014	rdev->pm.dpm.thermal.max_temp = high_temp;
1015
1016	return 0;
1017}
1018
1019static void trinity_update_current_ps(struct radeon_device *rdev,
1020				      struct radeon_ps *rps)
1021{
1022	struct trinity_ps *new_ps = trinity_get_ps(rps);
1023	struct trinity_power_info *pi = trinity_get_pi(rdev);
1024
1025	pi->current_rps = *rps;
1026	pi->current_ps = *new_ps;
1027	pi->current_rps.ps_priv = &pi->current_ps;
1028}
1029
1030static void trinity_update_requested_ps(struct radeon_device *rdev,
1031					struct radeon_ps *rps)
1032{
1033	struct trinity_ps *new_ps = trinity_get_ps(rps);
1034	struct trinity_power_info *pi = trinity_get_pi(rdev);
1035
1036	pi->requested_rps = *rps;
1037	pi->requested_ps = *new_ps;
1038	pi->requested_rps.ps_priv = &pi->requested_ps;
1039}
1040
1041void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1042{
1043	struct trinity_power_info *pi = trinity_get_pi(rdev);
1044
1045	if (pi->enable_bapm) {
1046		trinity_acquire_mutex(rdev);
1047		trinity_dpm_bapm_enable(rdev, enable);
1048		trinity_release_mutex(rdev);
1049	}
1050}
1051
1052int trinity_dpm_enable(struct radeon_device *rdev)
1053{
1054	struct trinity_power_info *pi = trinity_get_pi(rdev);
1055
1056	trinity_acquire_mutex(rdev);
1057
1058	if (trinity_dpm_enabled(rdev)) {
1059		trinity_release_mutex(rdev);
1060		return -EINVAL;
1061	}
1062
1063	trinity_program_bootup_state(rdev);
1064	sumo_program_vc(rdev, 0x00C00033);
1065	trinity_start_am(rdev);
1066	if (pi->enable_auto_thermal_throttling) {
1067		trinity_program_ttt(rdev);
1068		trinity_enable_att(rdev);
1069	}
1070	trinity_program_sclk_dpm(rdev);
1071	trinity_start_dpm(rdev);
1072	trinity_wait_for_dpm_enabled(rdev);
1073	trinity_dpm_bapm_enable(rdev, false);
1074	trinity_release_mutex(rdev);
1075
1076	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1077
1078	return 0;
1079}
1080
1081int trinity_dpm_late_enable(struct radeon_device *rdev)
1082{
1083	int ret;
1084
1085	trinity_acquire_mutex(rdev);
1086	trinity_enable_clock_power_gating(rdev);
1087
1088	if (rdev->irq.installed &&
1089	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1090		ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1091		if (ret) {
1092			trinity_release_mutex(rdev);
1093			return ret;
1094		}
1095		rdev->irq.dpm_thermal = true;
1096		radeon_irq_set(rdev);
1097	}
1098	trinity_release_mutex(rdev);
1099
1100	return 0;
1101}
1102
1103void trinity_dpm_disable(struct radeon_device *rdev)
1104{
1105	trinity_acquire_mutex(rdev);
1106	if (!trinity_dpm_enabled(rdev)) {
1107		trinity_release_mutex(rdev);
1108		return;
1109	}
1110	trinity_dpm_bapm_enable(rdev, false);
1111	trinity_disable_clock_power_gating(rdev);
1112	sumo_clear_vc(rdev);
1113	trinity_wait_for_level_0(rdev);
1114	trinity_stop_dpm(rdev);
1115	trinity_reset_am(rdev);
1116	trinity_release_mutex(rdev);
1117
1118	if (rdev->irq.installed &&
1119	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1120		rdev->irq.dpm_thermal = false;
1121		radeon_irq_set(rdev);
1122	}
1123
1124	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1125}
1126
1127static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1128{
1129	struct trinity_power_info *pi = trinity_get_pi(rdev);
1130
1131	pi->min_sclk_did =
1132		(RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1133}
1134
1135static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1136				  struct radeon_ps *rps)
1137{
1138	struct trinity_power_info *pi = trinity_get_pi(rdev);
1139	struct trinity_ps *new_ps = trinity_get_ps(rps);
1140	u32 nbpsconfig;
1141
1142	if (pi->sys_info.nb_dpm_enable) {
1143		nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1144		nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1145		nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1146			       Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1147			       DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1148			       DpmXNbPsHi(new_ps->DpmXNbPsHi));
1149		WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1150	}
1151}
1152
1153int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1154					enum radeon_dpm_forced_level level)
1155{
1156	struct trinity_power_info *pi = trinity_get_pi(rdev);
1157	struct radeon_ps *rps = &pi->current_rps;
1158	struct trinity_ps *ps = trinity_get_ps(rps);
1159	int i, ret;
1160
1161	if (ps->num_levels <= 1)
1162		return 0;
1163
1164	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1165		/* not supported by the hw */
1166		return -EINVAL;
1167	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1168		ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1169		if (ret)
1170			return ret;
1171	} else {
1172		for (i = 0; i < ps->num_levels; i++) {
1173			ret = trinity_dpm_n_levels_disabled(rdev, 0);
1174			if (ret)
1175				return ret;
1176		}
1177	}
1178
1179	rdev->pm.dpm.forced_level = level;
1180
1181	return 0;
1182}
1183
1184int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1185{
1186	struct trinity_power_info *pi = trinity_get_pi(rdev);
1187	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1188	struct radeon_ps *new_ps = &requested_ps;
1189
1190	trinity_update_requested_ps(rdev, new_ps);
1191
1192	trinity_apply_state_adjust_rules(rdev,
1193					 &pi->requested_rps,
1194					 &pi->current_rps);
1195
1196	return 0;
1197}
1198
1199int trinity_dpm_set_power_state(struct radeon_device *rdev)
1200{
1201	struct trinity_power_info *pi = trinity_get_pi(rdev);
1202	struct radeon_ps *new_ps = &pi->requested_rps;
1203	struct radeon_ps *old_ps = &pi->current_rps;
1204
1205	trinity_acquire_mutex(rdev);
1206	if (pi->enable_dpm) {
1207		if (pi->enable_bapm)
1208			trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1209		trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1210		trinity_enable_power_level_0(rdev);
1211		trinity_force_level_0(rdev);
1212		trinity_wait_for_level_0(rdev);
1213		trinity_setup_nbp_sim(rdev, new_ps);
1214		trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1215		trinity_force_level_0(rdev);
1216		trinity_unforce_levels(rdev);
1217		trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1218		trinity_set_vce_clock(rdev, new_ps, old_ps);
1219	}
1220	trinity_release_mutex(rdev);
1221
1222	return 0;
1223}
1224
1225void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1226{
1227	struct trinity_power_info *pi = trinity_get_pi(rdev);
1228	struct radeon_ps *new_ps = &pi->requested_rps;
1229
1230	trinity_update_current_ps(rdev, new_ps);
1231}
1232
1233void trinity_dpm_setup_asic(struct radeon_device *rdev)
1234{
1235	trinity_acquire_mutex(rdev);
1236	sumo_program_sstp(rdev);
1237	sumo_take_smu_control(rdev, true);
1238	trinity_get_min_sclk_divider(rdev);
1239	trinity_release_mutex(rdev);
1240}
1241
1242#if 0
1243void trinity_dpm_reset_asic(struct radeon_device *rdev)
1244{
1245	struct trinity_power_info *pi = trinity_get_pi(rdev);
1246
1247	trinity_acquire_mutex(rdev);
1248	if (pi->enable_dpm) {
1249		trinity_enable_power_level_0(rdev);
1250		trinity_force_level_0(rdev);
1251		trinity_wait_for_level_0(rdev);
1252		trinity_program_bootup_state(rdev);
1253		trinity_force_level_0(rdev);
1254		trinity_unforce_levels(rdev);
1255	}
1256	trinity_release_mutex(rdev);
1257}
1258#endif
1259
1260static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1261						  u32 vid_2bit)
1262{
1263	struct trinity_power_info *pi = trinity_get_pi(rdev);
1264	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1265	u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1266	u32 step = (svi_mode == 0) ? 1250 : 625;
1267	u32 delta = vid_7bit * step + 50;
1268
1269	if (delta > 155000)
1270		return 0;
1271
1272	return (155000 - delta) / 100;
1273}
1274
1275static void trinity_patch_boot_state(struct radeon_device *rdev,
1276				     struct trinity_ps *ps)
1277{
1278	struct trinity_power_info *pi = trinity_get_pi(rdev);
1279
1280	ps->num_levels = 1;
1281	ps->nbps_flags = 0;
1282	ps->bapm_flags = 0;
1283	ps->levels[0] = pi->boot_pl;
1284}
1285
1286static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1287{
1288	if (sclk < 20000)
1289		return 1;
1290	return 0;
1291}
1292
1293static void trinity_construct_boot_state(struct radeon_device *rdev)
1294{
1295	struct trinity_power_info *pi = trinity_get_pi(rdev);
1296
1297	pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1298	pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1299	pi->boot_pl.ds_divider_index = 0;
1300	pi->boot_pl.ss_divider_index = 0;
1301	pi->boot_pl.allow_gnb_slow = 1;
1302	pi->boot_pl.force_nbp_state = 0;
1303	pi->boot_pl.display_wm = 0;
1304	pi->boot_pl.vce_wm = 0;
1305	pi->current_ps.num_levels = 1;
1306	pi->current_ps.levels[0] = pi->boot_pl;
1307}
1308
1309static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1310						  u32 sclk, u32 min_sclk_in_sr)
1311{
1312	struct trinity_power_info *pi = trinity_get_pi(rdev);
1313	u32 i;
1314	u32 temp;
1315	u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1316		min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1317
1318	if (sclk < min)
1319		return 0;
1320
1321	if (!pi->enable_sclk_ds)
1322		return 0;
1323
1324	for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1325		temp = sclk / sumo_get_sleep_divider_from_id(i);
1326		if (temp >= min || i == 0)
1327			break;
1328	}
1329
1330	return (u8)i;
1331}
1332
1333static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1334					  u32 lower_limit)
1335{
1336	struct trinity_power_info *pi = trinity_get_pi(rdev);
1337	u32 i;
1338
1339	for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1340		if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1341			return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1342	}
1343
1344	if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1345		DRM_ERROR("engine clock out of range!");
1346
1347	return 0;
1348}
1349
1350static void trinity_patch_thermal_state(struct radeon_device *rdev,
1351					struct trinity_ps *ps,
1352					struct trinity_ps *current_ps)
1353{
1354	struct trinity_power_info *pi = trinity_get_pi(rdev);
1355	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1356	u32 current_vddc;
1357	u32 current_sclk;
1358	u32 current_index = 0;
1359
1360	if (current_ps) {
1361		current_vddc = current_ps->levels[current_index].vddc_index;
1362		current_sclk = current_ps->levels[current_index].sclk;
1363	} else {
1364		current_vddc = pi->boot_pl.vddc_index;
1365		current_sclk = pi->boot_pl.sclk;
1366	}
1367
1368	ps->levels[0].vddc_index = current_vddc;
1369
1370	if (ps->levels[0].sclk > current_sclk)
1371		ps->levels[0].sclk = current_sclk;
1372
1373	ps->levels[0].ds_divider_index =
1374		trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1375	ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1376	ps->levels[0].allow_gnb_slow = 1;
1377	ps->levels[0].force_nbp_state = 0;
1378	ps->levels[0].display_wm = 0;
1379	ps->levels[0].vce_wm =
1380		trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1381}
1382
1383static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1384				       struct trinity_ps *ps, u32 index)
1385{
1386	if (ps == NULL || ps->num_levels <= 1)
1387		return 0;
1388	else if (ps->num_levels == 2) {
1389		if (index == 0)
1390			return 0;
1391		else
1392			return 1;
1393	} else {
1394		if (index == 0)
1395			return 0;
1396		else if (ps->levels[index].sclk < 30000)
1397			return 0;
1398		else
1399			return 1;
1400	}
1401}
1402
1403static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1404				       struct radeon_ps *rps)
1405{
1406	struct trinity_power_info *pi = trinity_get_pi(rdev);
1407	u32 i = 0;
1408
1409	for (i = 0; i < 4; i++) {
1410		if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1411		    (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1412		    break;
1413	}
1414
1415	if (i >= 4) {
1416		DRM_ERROR("UVD clock index not found!\n");
1417		i = 3;
1418	}
1419	return i;
1420}
1421
1422static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1423				     struct radeon_ps *rps)
1424{
1425	struct trinity_ps *ps = trinity_get_ps(rps);
1426	struct trinity_power_info *pi = trinity_get_pi(rdev);
1427	u32 high_index = 0;
1428	u32 low_index = 0;
1429
1430	if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1431		high_index = trinity_get_uvd_clock_index(rdev, rps);
1432
1433		switch (high_index) {
1434		case 3:
1435		case 2:
1436			low_index = 1;
1437			break;
1438		case 1:
1439		case 0:
1440		default:
1441			low_index = 0;
1442			break;
1443		}
1444
1445		ps->vclk_low_divider =
1446			pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1447		ps->dclk_low_divider =
1448			pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1449		ps->vclk_high_divider =
1450			pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1451		ps->dclk_high_divider =
1452			pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1453	}
1454}
1455
1456static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1457					 u32 evclk, u32 ecclk, u16 *voltage)
1458{
1459	u32 i;
1460	int ret = -EINVAL;
1461	struct radeon_vce_clock_voltage_dependency_table *table =
1462		&rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1463
1464	if (((evclk == 0) && (ecclk == 0)) ||
1465	    (table && (table->count == 0))) {
1466		*voltage = 0;
1467		return 0;
1468	}
1469
1470	for (i = 0; i < table->count; i++) {
1471		if ((evclk <= table->entries[i].evclk) &&
1472		    (ecclk <= table->entries[i].ecclk)) {
1473			*voltage = table->entries[i].v;
1474			ret = 0;
1475			break;
1476		}
1477	}
1478
1479	/* if no match return the highest voltage */
1480	if (ret)
1481		*voltage = table->entries[table->count - 1].v;
1482
1483	return ret;
1484}
1485
1486static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1487					     struct radeon_ps *new_rps,
1488					     struct radeon_ps *old_rps)
1489{
1490	struct trinity_ps *ps = trinity_get_ps(new_rps);
1491	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1492	struct trinity_power_info *pi = trinity_get_pi(rdev);
1493	u32 min_voltage = 0; /* ??? */
1494	u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1495	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1496	u32 i;
1497	u16 min_vce_voltage;
1498	bool force_high;
1499	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1500
1501	if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1502		return trinity_patch_thermal_state(rdev, ps, current_ps);
1503
1504	trinity_adjust_uvd_state(rdev, new_rps);
1505
1506	if (new_rps->vce_active) {
1507		new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1508		new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1509	} else {
1510		new_rps->evclk = 0;
1511		new_rps->ecclk = 0;
1512	}
1513
1514	for (i = 0; i < ps->num_levels; i++) {
1515		if (ps->levels[i].vddc_index < min_voltage)
1516			ps->levels[i].vddc_index = min_voltage;
1517
1518		if (ps->levels[i].sclk < min_sclk)
1519			ps->levels[i].sclk =
1520				trinity_get_valid_engine_clock(rdev, min_sclk);
1521
1522		/* patch in vce limits */
1523		if (new_rps->vce_active) {
1524			/* sclk */
1525			if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1526				ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1527			/* vddc */
1528			trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1529			if (ps->levels[i].vddc_index < min_vce_voltage)
1530				ps->levels[i].vddc_index = min_vce_voltage;
1531		}
1532
1533		ps->levels[i].ds_divider_index =
1534			sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1535
1536		ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1537
1538		ps->levels[i].allow_gnb_slow = 1;
1539		ps->levels[i].force_nbp_state = 0;
1540		ps->levels[i].display_wm =
1541			trinity_calculate_display_wm(rdev, ps, i);
1542		ps->levels[i].vce_wm =
1543			trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1544	}
1545
1546	if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1547	    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1548		ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1549
1550	if (pi->sys_info.nb_dpm_enable) {
1551		ps->Dpm0PgNbPsLo = 0x1;
1552		ps->Dpm0PgNbPsHi = 0x0;
1553		ps->DpmXNbPsLo = 0x2;
1554		ps->DpmXNbPsHi = 0x1;
1555
1556		if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1557		    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1558			force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1559				      ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1560				       (pi->sys_info.uma_channel_number == 1)));
1561			force_high = (num_active_displays >= 3) || force_high;
1562			ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1563			ps->Dpm0PgNbPsHi = 0x1;
1564			ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1565			ps->DpmXNbPsHi = 0x2;
1566			ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1567		}
1568	}
1569}
1570
1571static void trinity_cleanup_asic(struct radeon_device *rdev)
1572{
1573	sumo_take_smu_control(rdev, false);
1574}
1575
1576#if 0
1577static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1578{
1579	struct trinity_power_info *pi = trinity_get_pi(rdev);
1580
1581	if (pi->voltage_drop_in_dce)
1582		trinity_dce_enable_voltage_adjustment(rdev, false);
1583}
1584#endif
1585
1586static void trinity_add_dccac_value(struct radeon_device *rdev)
1587{
1588	u32 gpu_cac_avrg_cntl_window_size;
1589	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1590	u64 disp_clk = rdev->clock.default_dispclk / 100;
1591	u32 dc_cac_value;
1592
1593	gpu_cac_avrg_cntl_window_size =
1594		(RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1595
1596	dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1597			     (32 - gpu_cac_avrg_cntl_window_size));
1598
1599	WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1600}
1601
1602void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1603{
1604	struct trinity_power_info *pi = trinity_get_pi(rdev);
1605
1606	if (pi->voltage_drop_in_dce)
1607		trinity_dce_enable_voltage_adjustment(rdev, true);
1608	trinity_add_dccac_value(rdev);
1609}
1610
1611union power_info {
1612	struct _ATOM_POWERPLAY_INFO info;
1613	struct _ATOM_POWERPLAY_INFO_V2 info_2;
1614	struct _ATOM_POWERPLAY_INFO_V3 info_3;
1615	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1616	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1617	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1618};
1619
1620union pplib_clock_info {
1621	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1622	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1623	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1624	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1625};
1626
1627union pplib_power_state {
1628	struct _ATOM_PPLIB_STATE v1;
1629	struct _ATOM_PPLIB_STATE_V2 v2;
1630};
1631
1632static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1633					       struct radeon_ps *rps,
1634					       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1635					       u8 table_rev)
1636{
1637	struct trinity_ps *ps = trinity_get_ps(rps);
1638
1639	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1640	rps->class = le16_to_cpu(non_clock_info->usClassification);
1641	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1642
1643	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1644		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1645		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1646	} else {
1647		rps->vclk = 0;
1648		rps->dclk = 0;
1649	}
1650
1651	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1652		rdev->pm.dpm.boot_ps = rps;
1653		trinity_patch_boot_state(rdev, ps);
1654	}
1655	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1656		rdev->pm.dpm.uvd_ps = rps;
1657}
1658
1659static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1660					   struct radeon_ps *rps, int index,
1661					   union pplib_clock_info *clock_info)
1662{
1663	struct trinity_power_info *pi = trinity_get_pi(rdev);
1664	struct trinity_ps *ps = trinity_get_ps(rps);
1665	struct trinity_pl *pl = &ps->levels[index];
1666	u32 sclk;
1667
1668	sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1669	sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1670	pl->sclk = sclk;
1671	pl->vddc_index = clock_info->sumo.vddcIndex;
1672
1673	ps->num_levels = index + 1;
1674
1675	if (pi->enable_sclk_ds) {
1676		pl->ds_divider_index = 5;
1677		pl->ss_divider_index = 5;
1678	}
1679}
1680
1681static int trinity_parse_power_table(struct radeon_device *rdev)
1682{
1683	struct radeon_mode_info *mode_info = &rdev->mode_info;
1684	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1685	union pplib_power_state *power_state;
1686	int i, j, k, non_clock_array_index, clock_array_index;
1687	union pplib_clock_info *clock_info;
1688	struct _StateArray *state_array;
1689	struct _ClockInfoArray *clock_info_array;
1690	struct _NonClockInfoArray *non_clock_info_array;
1691	union power_info *power_info;
1692	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1693	u16 data_offset;
1694	u8 frev, crev;
1695	u8 *power_state_offset;
1696	struct sumo_ps *ps;
1697
1698	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1699				   &frev, &crev, &data_offset))
1700		return -EINVAL;
1701	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1702
1703	state_array = (struct _StateArray *)
1704		(mode_info->atom_context->bios + data_offset +
1705		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
1706	clock_info_array = (struct _ClockInfoArray *)
1707		(mode_info->atom_context->bios + data_offset +
1708		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1709	non_clock_info_array = (struct _NonClockInfoArray *)
1710		(mode_info->atom_context->bios + data_offset +
1711		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1712
1713	rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1714				  sizeof(struct radeon_ps),
1715				  GFP_KERNEL);
1716	if (!rdev->pm.dpm.ps)
1717		return -ENOMEM;
1718	power_state_offset = (u8 *)state_array->states;
1719	for (i = 0; i < state_array->ucNumEntries; i++) {
1720		u8 *idx;
1721		power_state = (union pplib_power_state *)power_state_offset;
1722		non_clock_array_index = power_state->v2.nonClockInfoIndex;
1723		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1724			&non_clock_info_array->nonClockInfo[non_clock_array_index];
1725		if (!rdev->pm.power_state[i].clock_info) {
1726			kfree(rdev->pm.dpm.ps);
1727			return -EINVAL;
1728		}
1729		ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1730		if (ps == NULL) {
1731			kfree(rdev->pm.dpm.ps);
1732			return -ENOMEM;
1733		}
1734		rdev->pm.dpm.ps[i].ps_priv = ps;
1735		k = 0;
1736		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1737		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1738			clock_array_index = idx[j];
1739			if (clock_array_index >= clock_info_array->ucNumEntries)
1740				continue;
1741			if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1742				break;
1743			clock_info = (union pplib_clock_info *)
1744				((u8 *)&clock_info_array->clockInfo[0] +
1745				 (clock_array_index * clock_info_array->ucEntrySize));
1746			trinity_parse_pplib_clock_info(rdev,
1747						       &rdev->pm.dpm.ps[i], k,
1748						       clock_info);
1749			k++;
1750		}
1751		trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1752						   non_clock_info,
1753						   non_clock_info_array->ucEntrySize);
1754		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1755	}
1756	rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1757
1758	/* fill in the vce power states */
1759	for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1760		u32 sclk;
1761		clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1762		clock_info = (union pplib_clock_info *)
1763			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1764		sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1765		sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1766		rdev->pm.dpm.vce_states[i].sclk = sclk;
1767		rdev->pm.dpm.vce_states[i].mclk = 0;
1768	}
1769
1770	return 0;
1771}
1772
1773union igp_info {
1774	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1775	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1776	struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1777	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1778	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1779};
1780
1781static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1782{
1783	struct trinity_power_info *pi = trinity_get_pi(rdev);
1784	u32 divider;
1785
1786	if (did >= 8 && did <= 0x3f)
1787		divider = did * 25;
1788	else if (did > 0x3f && did <= 0x5f)
1789		divider = (did - 64) * 50 + 1600;
1790	else if (did > 0x5f && did <= 0x7e)
1791		divider = (did - 96) * 100 + 3200;
1792	else if (did == 0x7f)
1793		divider = 128 * 100;
1794	else
1795		return 10000;
1796
1797	return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1798}
1799
1800static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1801{
1802	struct trinity_power_info *pi = trinity_get_pi(rdev);
1803	struct radeon_mode_info *mode_info = &rdev->mode_info;
1804	int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1805	union igp_info *igp_info;
1806	u8 frev, crev;
1807	u16 data_offset;
1808	int i;
1809
1810	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1811				   &frev, &crev, &data_offset)) {
1812		igp_info = (union igp_info *)(mode_info->atom_context->bios +
1813					      data_offset);
1814
1815		if (crev != 7) {
1816			DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1817			return -EINVAL;
1818		}
1819		pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1820		pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1821		pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1822		pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1823		pi->sys_info.bootup_nb_voltage_index =
1824			le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1825		if (igp_info->info_7.ucHtcTmpLmt == 0)
1826			pi->sys_info.htc_tmp_lmt = 203;
1827		else
1828			pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1829		if (igp_info->info_7.ucHtcHystLmt == 0)
1830			pi->sys_info.htc_hyst_lmt = 5;
1831		else
1832			pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1833		if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1834			DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1835		}
1836
1837		if (pi->enable_nbps_policy)
1838			pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1839		else
1840			pi->sys_info.nb_dpm_enable = 0;
1841
1842		for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1843			pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1844			pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1845		}
1846
1847		pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1848		pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1849		pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1850		pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1851
1852		if (!pi->sys_info.nb_dpm_enable) {
1853			for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1854				pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1855				pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1856				pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1857			}
1858		}
1859
1860		pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1861
1862		sumo_construct_sclk_voltage_mapping_table(rdev,
1863							  &pi->sys_info.sclk_voltage_mapping_table,
1864							  igp_info->info_7.sAvail_SCLK);
1865		sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1866						 igp_info->info_7.sAvail_SCLK);
1867
1868		pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1869			igp_info->info_7.ucDPMState0VclkFid;
1870		pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1871			igp_info->info_7.ucDPMState1VclkFid;
1872		pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1873			igp_info->info_7.ucDPMState2VclkFid;
1874		pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1875			igp_info->info_7.ucDPMState3VclkFid;
1876
1877		pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1878			igp_info->info_7.ucDPMState0DclkFid;
1879		pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1880			igp_info->info_7.ucDPMState1DclkFid;
1881		pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1882			igp_info->info_7.ucDPMState2DclkFid;
1883		pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1884			igp_info->info_7.ucDPMState3DclkFid;
1885
1886		for (i = 0; i < 4; i++) {
1887			pi->sys_info.uvd_clock_table_entries[i].vclk =
1888				trinity_convert_did_to_freq(rdev,
1889							    pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1890			pi->sys_info.uvd_clock_table_entries[i].dclk =
1891				trinity_convert_did_to_freq(rdev,
1892							    pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1893		}
1894
1895
1896
1897	}
1898	return 0;
1899}
1900
1901int trinity_dpm_init(struct radeon_device *rdev)
1902{
1903	struct trinity_power_info *pi;
1904	int ret, i;
1905
1906	pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1907	if (pi == NULL)
1908		return -ENOMEM;
1909	rdev->pm.dpm.priv = pi;
1910
1911	for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1912		pi->at[i] = TRINITY_AT_DFLT;
1913
1914	if (radeon_bapm == -1) {
1915		/* There are stability issues reported on with
1916		 * bapm enabled when switching between AC and battery
1917		 * power.  At the same time, some MSI boards hang
1918		 * if it's not enabled and dpm is enabled.  Just enable
1919		 * it for MSI boards right now.
1920		 */
1921		if (rdev->pdev->subsystem_vendor == 0x1462)
1922			pi->enable_bapm = true;
1923		else
1924			pi->enable_bapm = false;
1925	} else if (radeon_bapm == 0) {
1926		pi->enable_bapm = false;
1927	} else {
1928		pi->enable_bapm = true;
1929	}
1930	pi->enable_nbps_policy = true;
1931	pi->enable_sclk_ds = true;
1932	pi->enable_gfx_power_gating = true;
1933	pi->enable_gfx_clock_gating = true;
1934	pi->enable_mg_clock_gating = false;
1935	pi->enable_gfx_dynamic_mgpg = false;
1936	pi->override_dynamic_mgpg = false;
1937	pi->enable_auto_thermal_throttling = true;
1938	pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1939	pi->uvd_dpm = true; /* ??? */
1940
1941	ret = trinity_parse_sys_info_table(rdev);
1942	if (ret)
1943		return ret;
1944
1945	trinity_construct_boot_state(rdev);
1946
1947	ret = r600_get_platform_caps(rdev);
1948	if (ret)
1949		return ret;
1950
1951	ret = r600_parse_extended_power_table(rdev);
1952	if (ret)
1953		return ret;
1954
1955	ret = trinity_parse_power_table(rdev);
1956	if (ret)
1957		return ret;
1958
1959	pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
1960	pi->enable_dpm = true;
1961
1962	return 0;
1963}
1964
1965void trinity_dpm_print_power_state(struct radeon_device *rdev,
1966				   struct radeon_ps *rps)
1967{
1968	int i;
1969	struct trinity_ps *ps = trinity_get_ps(rps);
1970
1971	r600_dpm_print_class_info(rps->class, rps->class2);
1972	r600_dpm_print_cap_info(rps->caps);
1973	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1974	for (i = 0; i < ps->num_levels; i++) {
1975		struct trinity_pl *pl = &ps->levels[i];
1976		printk("\t\tpower level %d    sclk: %u vddc: %u\n",
1977		       i, pl->sclk,
1978		       trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
1979	}
1980	r600_dpm_print_ps_status(rdev, rps);
1981}
1982
1983void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
1984							 struct seq_file *m)
1985{
1986	struct trinity_power_info *pi = trinity_get_pi(rdev);
1987	struct radeon_ps *rps = &pi->current_rps;
1988	struct trinity_ps *ps = trinity_get_ps(rps);
1989	struct trinity_pl *pl;
1990	u32 current_index =
1991		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
1992		CURRENT_STATE_SHIFT;
1993
1994	if (current_index >= ps->num_levels) {
1995		seq_printf(m, "invalid dpm profile %d\n", current_index);
1996	} else {
1997		pl = &ps->levels[current_index];
1998		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1999		seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2000			   current_index, pl->sclk,
2001			   trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2002	}
2003}
2004
2005u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2006{
2007	struct trinity_power_info *pi = trinity_get_pi(rdev);
2008	struct radeon_ps *rps = &pi->current_rps;
2009	struct trinity_ps *ps = trinity_get_ps(rps);
2010	struct trinity_pl *pl;
2011	u32 current_index =
2012		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2013		CURRENT_STATE_SHIFT;
2014
2015	if (current_index >= ps->num_levels) {
2016		return 0;
2017	} else {
2018		pl = &ps->levels[current_index];
2019		return pl->sclk;
2020	}
2021}
2022
2023u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2024{
2025	struct trinity_power_info *pi = trinity_get_pi(rdev);
2026
2027	return pi->sys_info.bootup_uma_clk;
2028}
2029
2030void trinity_dpm_fini(struct radeon_device *rdev)
2031{
2032	int i;
2033
2034	trinity_cleanup_asic(rdev); /* ??? */
2035
2036	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2037		kfree(rdev->pm.dpm.ps[i].ps_priv);
2038	}
2039	kfree(rdev->pm.dpm.ps);
2040	kfree(rdev->pm.dpm.priv);
2041	r600_free_extended_power_table(rdev);
2042}
2043
2044u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2045{
2046	struct trinity_power_info *pi = trinity_get_pi(rdev);
2047	struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2048
2049	if (low)
2050		return requested_state->levels[0].sclk;
2051	else
2052		return requested_state->levels[requested_state->num_levels - 1].sclk;
2053}
2054
2055u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2056{
2057	struct trinity_power_info *pi = trinity_get_pi(rdev);
2058
2059	return pi->sys_info.bootup_uma_clk;
2060}