1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
 
  24#include <linux/seq_file.h>
  25
  26#include <drm/drm_pci.h>
  27
  28#include "r600_dpm.h"
  29#include "radeon.h"
  30#include "radeon_asic.h"
  31#include "trinity_dpm.h"
  32#include "trinityd.h"
 
  33
  34#define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
  35#define TRINITY_MINIMUM_ENGINE_CLOCK 800
  36#define SCLK_MIN_DIV_INTV_SHIFT     12
  37#define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
  38
  39#ifndef TRINITY_MGCG_SEQUENCE
  40#define TRINITY_MGCG_SEQUENCE  100
  41
  42static const u32 trinity_mgcg_shls_default[] =
  43{
  44	/* Register, Value, Mask */
  45	0x0000802c, 0xc0000000, 0xffffffff,
  46	0x00003fc4, 0xc0000000, 0xffffffff,
  47	0x00005448, 0x00000100, 0xffffffff,
  48	0x000055e4, 0x00000100, 0xffffffff,
  49	0x0000160c, 0x00000100, 0xffffffff,
  50	0x00008984, 0x06000100, 0xffffffff,
  51	0x0000c164, 0x00000100, 0xffffffff,
  52	0x00008a18, 0x00000100, 0xffffffff,
  53	0x0000897c, 0x06000100, 0xffffffff,
  54	0x00008b28, 0x00000100, 0xffffffff,
  55	0x00009144, 0x00800200, 0xffffffff,
  56	0x00009a60, 0x00000100, 0xffffffff,
  57	0x00009868, 0x00000100, 0xffffffff,
  58	0x00008d58, 0x00000100, 0xffffffff,
  59	0x00009510, 0x00000100, 0xffffffff,
  60	0x0000949c, 0x00000100, 0xffffffff,
  61	0x00009654, 0x00000100, 0xffffffff,
  62	0x00009030, 0x00000100, 0xffffffff,
  63	0x00009034, 0x00000100, 0xffffffff,
  64	0x00009038, 0x00000100, 0xffffffff,
  65	0x0000903c, 0x00000100, 0xffffffff,
  66	0x00009040, 0x00000100, 0xffffffff,
  67	0x0000a200, 0x00000100, 0xffffffff,
  68	0x0000a204, 0x00000100, 0xffffffff,
  69	0x0000a208, 0x00000100, 0xffffffff,
  70	0x0000a20c, 0x00000100, 0xffffffff,
  71	0x00009744, 0x00000100, 0xffffffff,
  72	0x00003f80, 0x00000100, 0xffffffff,
  73	0x0000a210, 0x00000100, 0xffffffff,
  74	0x0000a214, 0x00000100, 0xffffffff,
  75	0x000004d8, 0x00000100, 0xffffffff,
  76	0x00009664, 0x00000100, 0xffffffff,
  77	0x00009698, 0x00000100, 0xffffffff,
  78	0x000004d4, 0x00000200, 0xffffffff,
  79	0x000004d0, 0x00000000, 0xffffffff,
  80	0x000030cc, 0x00000104, 0xffffffff,
  81	0x0000d0c0, 0x00000100, 0xffffffff,
  82	0x0000d8c0, 0x00000100, 0xffffffff,
  83	0x0000951c, 0x00010000, 0xffffffff,
  84	0x00009160, 0x00030002, 0xffffffff,
  85	0x00009164, 0x00050004, 0xffffffff,
  86	0x00009168, 0x00070006, 0xffffffff,
  87	0x00009178, 0x00070000, 0xffffffff,
  88	0x0000917c, 0x00030002, 0xffffffff,
  89	0x00009180, 0x00050004, 0xffffffff,
  90	0x0000918c, 0x00010006, 0xffffffff,
  91	0x00009190, 0x00090008, 0xffffffff,
  92	0x00009194, 0x00070000, 0xffffffff,
  93	0x00009198, 0x00030002, 0xffffffff,
  94	0x0000919c, 0x00050004, 0xffffffff,
  95	0x000091a8, 0x00010006, 0xffffffff,
  96	0x000091ac, 0x00090008, 0xffffffff,
  97	0x000091b0, 0x00070000, 0xffffffff,
  98	0x000091b4, 0x00030002, 0xffffffff,
  99	0x000091b8, 0x00050004, 0xffffffff,
 100	0x000091c4, 0x00010006, 0xffffffff,
 101	0x000091c8, 0x00090008, 0xffffffff,
 102	0x000091cc, 0x00070000, 0xffffffff,
 103	0x000091d0, 0x00030002, 0xffffffff,
 104	0x000091d4, 0x00050004, 0xffffffff,
 105	0x000091e0, 0x00010006, 0xffffffff,
 106	0x000091e4, 0x00090008, 0xffffffff,
 107	0x000091e8, 0x00000000, 0xffffffff,
 108	0x000091ec, 0x00070000, 0xffffffff,
 109	0x000091f0, 0x00030002, 0xffffffff,
 110	0x000091f4, 0x00050004, 0xffffffff,
 111	0x00009200, 0x00010006, 0xffffffff,
 112	0x00009204, 0x00090008, 0xffffffff,
 113	0x00009208, 0x00070000, 0xffffffff,
 114	0x0000920c, 0x00030002, 0xffffffff,
 115	0x00009210, 0x00050004, 0xffffffff,
 116	0x0000921c, 0x00010006, 0xffffffff,
 117	0x00009220, 0x00090008, 0xffffffff,
 118	0x00009294, 0x00000000, 0xffffffff
 119};
 120
 121static const u32 trinity_mgcg_shls_enable[] =
 122{
 123	/* Register, Value, Mask */
 124	0x0000802c, 0xc0000000, 0xffffffff,
 125	0x000008f8, 0x00000000, 0xffffffff,
 126	0x000008fc, 0x00000000, 0x000133FF,
 127	0x000008f8, 0x00000001, 0xffffffff,
 128	0x000008fc, 0x00000000, 0xE00B03FC,
 129	0x00009150, 0x96944200, 0xffffffff
 130};
 131
 132static const u32 trinity_mgcg_shls_disable[] =
 133{
 134	/* Register, Value, Mask */
 135	0x0000802c, 0xc0000000, 0xffffffff,
 136	0x00009150, 0x00600000, 0xffffffff,
 137	0x000008f8, 0x00000000, 0xffffffff,
 138	0x000008fc, 0xffffffff, 0x000133FF,
 139	0x000008f8, 0x00000001, 0xffffffff,
 140	0x000008fc, 0xffffffff, 0xE00B03FC
 141};
 142#endif
 143
 144#ifndef TRINITY_SYSLS_SEQUENCE
 145#define TRINITY_SYSLS_SEQUENCE  100
 146
 147static const u32 trinity_sysls_default[] =
 148{
 149	/* Register, Value, Mask */
 150	0x000055e8, 0x00000000, 0xffffffff,
 151	0x0000d0bc, 0x00000000, 0xffffffff,
 152	0x0000d8bc, 0x00000000, 0xffffffff,
 153	0x000015c0, 0x000c1401, 0xffffffff,
 154	0x0000264c, 0x000c0400, 0xffffffff,
 155	0x00002648, 0x000c0400, 0xffffffff,
 156	0x00002650, 0x000c0400, 0xffffffff,
 157	0x000020b8, 0x000c0400, 0xffffffff,
 158	0x000020bc, 0x000c0400, 0xffffffff,
 159	0x000020c0, 0x000c0c80, 0xffffffff,
 160	0x0000f4a0, 0x000000c0, 0xffffffff,
 161	0x0000f4a4, 0x00680fff, 0xffffffff,
 162	0x00002f50, 0x00000404, 0xffffffff,
 163	0x000004c8, 0x00000001, 0xffffffff,
 164	0x0000641c, 0x00000000, 0xffffffff,
 165	0x00000c7c, 0x00000000, 0xffffffff,
 166	0x00006dfc, 0x00000000, 0xffffffff
 167};
 168
 169static const u32 trinity_sysls_disable[] =
 170{
 171	/* Register, Value, Mask */
 172	0x0000d0c0, 0x00000000, 0xffffffff,
 173	0x0000d8c0, 0x00000000, 0xffffffff,
 174	0x000055e8, 0x00000000, 0xffffffff,
 175	0x0000d0bc, 0x00000000, 0xffffffff,
 176	0x0000d8bc, 0x00000000, 0xffffffff,
 177	0x000015c0, 0x00041401, 0xffffffff,
 178	0x0000264c, 0x00040400, 0xffffffff,
 179	0x00002648, 0x00040400, 0xffffffff,
 180	0x00002650, 0x00040400, 0xffffffff,
 181	0x000020b8, 0x00040400, 0xffffffff,
 182	0x000020bc, 0x00040400, 0xffffffff,
 183	0x000020c0, 0x00040c80, 0xffffffff,
 184	0x0000f4a0, 0x000000c0, 0xffffffff,
 185	0x0000f4a4, 0x00680000, 0xffffffff,
 186	0x00002f50, 0x00000404, 0xffffffff,
 187	0x000004c8, 0x00000001, 0xffffffff,
 188	0x0000641c, 0x00007ffd, 0xffffffff,
 189	0x00000c7c, 0x0000ff00, 0xffffffff,
 190	0x00006dfc, 0x0000007f, 0xffffffff
 191};
 192
 193static const u32 trinity_sysls_enable[] =
 194{
 195	/* Register, Value, Mask */
 196	0x000055e8, 0x00000001, 0xffffffff,
 197	0x0000d0bc, 0x00000100, 0xffffffff,
 198	0x0000d8bc, 0x00000100, 0xffffffff,
 199	0x000015c0, 0x000c1401, 0xffffffff,
 200	0x0000264c, 0x000c0400, 0xffffffff,
 201	0x00002648, 0x000c0400, 0xffffffff,
 202	0x00002650, 0x000c0400, 0xffffffff,
 203	0x000020b8, 0x000c0400, 0xffffffff,
 204	0x000020bc, 0x000c0400, 0xffffffff,
 205	0x000020c0, 0x000c0c80, 0xffffffff,
 206	0x0000f4a0, 0x000000c0, 0xffffffff,
 207	0x0000f4a4, 0x00680fff, 0xffffffff,
 208	0x00002f50, 0x00000903, 0xffffffff,
 209	0x000004c8, 0x00000000, 0xffffffff,
 210	0x0000641c, 0x00000000, 0xffffffff,
 211	0x00000c7c, 0x00000000, 0xffffffff,
 212	0x00006dfc, 0x00000000, 0xffffffff
 213};
 214#endif
 215
 216static const u32 trinity_override_mgpg_sequences[] =
 217{
 218	/* Register, Value */
 219	0x00000200, 0xE030032C,
 220	0x00000204, 0x00000FFF,
 221	0x00000200, 0xE0300058,
 222	0x00000204, 0x00030301,
 223	0x00000200, 0xE0300054,
 224	0x00000204, 0x500010FF,
 225	0x00000200, 0xE0300074,
 226	0x00000204, 0x00030301,
 227	0x00000200, 0xE0300070,
 228	0x00000204, 0x500010FF,
 229	0x00000200, 0xE0300090,
 230	0x00000204, 0x00030301,
 231	0x00000200, 0xE030008C,
 232	0x00000204, 0x500010FF,
 233	0x00000200, 0xE03000AC,
 234	0x00000204, 0x00030301,
 235	0x00000200, 0xE03000A8,
 236	0x00000204, 0x500010FF,
 237	0x00000200, 0xE03000C8,
 238	0x00000204, 0x00030301,
 239	0x00000200, 0xE03000C4,
 240	0x00000204, 0x500010FF,
 241	0x00000200, 0xE03000E4,
 242	0x00000204, 0x00030301,
 243	0x00000200, 0xE03000E0,
 244	0x00000204, 0x500010FF,
 245	0x00000200, 0xE0300100,
 246	0x00000204, 0x00030301,
 247	0x00000200, 0xE03000FC,
 248	0x00000204, 0x500010FF,
 249	0x00000200, 0xE0300058,
 250	0x00000204, 0x00030303,
 251	0x00000200, 0xE0300054,
 252	0x00000204, 0x600010FF,
 253	0x00000200, 0xE0300074,
 254	0x00000204, 0x00030303,
 255	0x00000200, 0xE0300070,
 256	0x00000204, 0x600010FF,
 257	0x00000200, 0xE0300090,
 258	0x00000204, 0x00030303,
 259	0x00000200, 0xE030008C,
 260	0x00000204, 0x600010FF,
 261	0x00000200, 0xE03000AC,
 262	0x00000204, 0x00030303,
 263	0x00000200, 0xE03000A8,
 264	0x00000204, 0x600010FF,
 265	0x00000200, 0xE03000C8,
 266	0x00000204, 0x00030303,
 267	0x00000200, 0xE03000C4,
 268	0x00000204, 0x600010FF,
 269	0x00000200, 0xE03000E4,
 270	0x00000204, 0x00030303,
 271	0x00000200, 0xE03000E0,
 272	0x00000204, 0x600010FF,
 273	0x00000200, 0xE0300100,
 274	0x00000204, 0x00030303,
 275	0x00000200, 0xE03000FC,
 276	0x00000204, 0x600010FF,
 277	0x00000200, 0xE0300058,
 278	0x00000204, 0x00030303,
 279	0x00000200, 0xE0300054,
 280	0x00000204, 0x700010FF,
 281	0x00000200, 0xE0300074,
 282	0x00000204, 0x00030303,
 283	0x00000200, 0xE0300070,
 284	0x00000204, 0x700010FF,
 285	0x00000200, 0xE0300090,
 286	0x00000204, 0x00030303,
 287	0x00000200, 0xE030008C,
 288	0x00000204, 0x700010FF,
 289	0x00000200, 0xE03000AC,
 290	0x00000204, 0x00030303,
 291	0x00000200, 0xE03000A8,
 292	0x00000204, 0x700010FF,
 293	0x00000200, 0xE03000C8,
 294	0x00000204, 0x00030303,
 295	0x00000200, 0xE03000C4,
 296	0x00000204, 0x700010FF,
 297	0x00000200, 0xE03000E4,
 298	0x00000204, 0x00030303,
 299	0x00000200, 0xE03000E0,
 300	0x00000204, 0x700010FF,
 301	0x00000200, 0xE0300100,
 302	0x00000204, 0x00030303,
 303	0x00000200, 0xE03000FC,
 304	0x00000204, 0x700010FF,
 305	0x00000200, 0xE0300058,
 306	0x00000204, 0x00010303,
 307	0x00000200, 0xE0300054,
 308	0x00000204, 0x800010FF,
 309	0x00000200, 0xE0300074,
 310	0x00000204, 0x00010303,
 311	0x00000200, 0xE0300070,
 312	0x00000204, 0x800010FF,
 313	0x00000200, 0xE0300090,
 314	0x00000204, 0x00010303,
 315	0x00000200, 0xE030008C,
 316	0x00000204, 0x800010FF,
 317	0x00000200, 0xE03000AC,
 318	0x00000204, 0x00010303,
 319	0x00000200, 0xE03000A8,
 320	0x00000204, 0x800010FF,
 321	0x00000200, 0xE03000C4,
 322	0x00000204, 0x800010FF,
 323	0x00000200, 0xE03000C8,
 324	0x00000204, 0x00010303,
 325	0x00000200, 0xE03000E4,
 326	0x00000204, 0x00010303,
 327	0x00000200, 0xE03000E0,
 328	0x00000204, 0x800010FF,
 329	0x00000200, 0xE0300100,
 330	0x00000204, 0x00010303,
 331	0x00000200, 0xE03000FC,
 332	0x00000204, 0x800010FF,
 333	0x00000200, 0x0001f198,
 334	0x00000204, 0x0003ffff,
 335	0x00000200, 0x0001f19C,
 336	0x00000204, 0x3fffffff,
 337	0x00000200, 0xE030032C,
 338	0x00000204, 0x00000000,
 339};
 340
 341extern void vce_v1_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 342static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 343						   const u32 *seq, u32 count);
 344static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
 345static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
 346					     struct radeon_ps *new_rps,
 347					     struct radeon_ps *old_rps);
 348
 349static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
 350{
 351	struct trinity_ps *ps = rps->ps_priv;
 352
 353	return ps;
 354}
 355
 356static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
 357{
 358	struct trinity_power_info *pi = rdev->pm.dpm.priv;
 359
 360	return pi;
 361}
 362
 363static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
 364{
 365	struct trinity_power_info *pi = trinity_get_pi(rdev);
 366	u32 p, u;
 367	u32 value;
 368	struct atom_clock_dividers dividers;
 369	u32 xclk = radeon_get_xclk(rdev);
 370	u32 sssd = 1;
 371	int ret;
 372	u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
 373
 374	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 375					     25000, false, &dividers);
 376	if (ret)
 377		return;
 378
 379	value = RREG32_SMC(GFX_POWER_GATING_CNTL);
 380	value &= ~(SSSD_MASK | PDS_DIV_MASK);
 381	if (sssd)
 382		value |= SSSD(1);
 383	value |= PDS_DIV(dividers.post_div);
 384	WREG32_SMC(GFX_POWER_GATING_CNTL, value);
 385
 386	r600_calculate_u_and_p(500, xclk, 16, &p, &u);
 387
 388	WREG32(CG_PG_CTRL, SP(p) | SU(u));
 389
 390	WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
 391
 392	/* XXX double check hw_rev */
 393	if (pi->override_dynamic_mgpg && (hw_rev == 0))
 394		trinity_override_dynamic_mg_powergating(rdev);
 395
 396}
 397
 398#define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
 399#define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
 400#define CGTS_SM_CTRL_REG_DISABLE    0x00600000
 401#define CGTS_SM_CTRL_REG_ENABLE     0x96944200
 402
 403static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
 404					  bool enable)
 405{
 406	u32 local0;
 407	u32 local1;
 408
 409	if (enable) {
 410		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 411		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 412
 413		WREG32_CG(CG_CGTT_LOCAL_0,
 414			  (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 415		WREG32_CG(CG_CGTT_LOCAL_1,
 416			  (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 417
 418		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
 419	} else {
 420		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
 421
 422		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 423		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 424
 425		WREG32_CG(CG_CGTT_LOCAL_0,
 426			  CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 427		WREG32_CG(CG_CGTT_LOCAL_1,
 428			  CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 429	}
 430}
 431
 432static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
 433{
 434	u32 count;
 435	const u32 *seq = NULL;
 436
 437	seq = &trinity_mgcg_shls_default[0];
 438	count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
 439
 440	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 441}
 442
 443static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
 444					   bool enable)
 445{
 446	if (enable) {
 447		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 448	} else {
 449		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 450		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 451		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 452		RREG32(GB_ADDR_CONFIG);
 453	}
 454}
 455
 456static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 457						   const u32 *seq, u32 count)
 458{
 459	u32 i, length = count * 3;
 460
 461	for (i = 0; i < length; i += 3)
 462		WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
 463}
 464
 465static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
 466						    const u32 *seq, u32 count)
 467{
 468	u32  i, length = count * 2;
 469
 470	for (i = 0; i < length; i += 2)
 471		WREG32(seq[i], seq[i+1]);
 472
 473}
 474
 475static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
 476{
 477	u32 count;
 478	const u32 *seq = NULL;
 479
 480	seq = &trinity_override_mgpg_sequences[0];
 481	count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
 482
 483	trinity_program_override_mgpg_sequences(rdev, seq, count);
 484}
 485
 486static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
 487					  bool enable)
 488{
 489	u32 count;
 490	const u32 *seq = NULL;
 491
 492	if (enable) {
 493		seq = &trinity_sysls_enable[0];
 494		count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
 495	} else {
 496		seq = &trinity_sysls_disable[0];
 497		count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
 498	}
 499
 500	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 501}
 502
 503static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
 504					   bool enable)
 505{
 506	if (enable) {
 507		if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
 508			WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
 509
 510		WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
 511	} else {
 512		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
 513		RREG32(GB_ADDR_CONFIG);
 514	}
 515}
 516
 517static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
 518					    bool enable)
 519{
 520	u32 value;
 521
 522	if (enable) {
 523		value = RREG32_SMC(PM_I_CNTL_1);
 524		value &= ~DS_PG_CNTL_MASK;
 525		value |= DS_PG_CNTL(1);
 526		WREG32_SMC(PM_I_CNTL_1, value);
 527
 528		value = RREG32_SMC(SMU_S_PG_CNTL);
 529		value &= ~DS_PG_EN_MASK;
 530		value |= DS_PG_EN(1);
 531		WREG32_SMC(SMU_S_PG_CNTL, value);
 532	} else {
 533		value = RREG32_SMC(SMU_S_PG_CNTL);
 534		value &= ~DS_PG_EN_MASK;
 535		WREG32_SMC(SMU_S_PG_CNTL, value);
 536
 537		value = RREG32_SMC(PM_I_CNTL_1);
 538		value &= ~DS_PG_CNTL_MASK;
 539		WREG32_SMC(PM_I_CNTL_1, value);
 540	}
 541
 542	trinity_gfx_dynamic_mgpg_config(rdev);
 543
 544}
 545
 546static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
 547{
 548	struct trinity_power_info *pi = trinity_get_pi(rdev);
 549
 550	if (pi->enable_gfx_clock_gating)
 551		sumo_gfx_clockgating_initialize(rdev);
 552	if (pi->enable_mg_clock_gating)
 553		trinity_mg_clockgating_initialize(rdev);
 554	if (pi->enable_gfx_power_gating)
 555		trinity_gfx_powergating_initialize(rdev);
 556	if (pi->enable_mg_clock_gating) {
 557		trinity_ls_clockgating_enable(rdev, true);
 558		trinity_mg_clockgating_enable(rdev, true);
 559	}
 560	if (pi->enable_gfx_clock_gating)
 561		trinity_gfx_clockgating_enable(rdev, true);
 562	if (pi->enable_gfx_dynamic_mgpg)
 563		trinity_gfx_dynamic_mgpg_enable(rdev, true);
 564	if (pi->enable_gfx_power_gating)
 565		trinity_gfx_powergating_enable(rdev, true);
 566}
 567
 568static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
 569{
 570	struct trinity_power_info *pi = trinity_get_pi(rdev);
 571
 572	if (pi->enable_gfx_power_gating)
 573		trinity_gfx_powergating_enable(rdev, false);
 574	if (pi->enable_gfx_dynamic_mgpg)
 575		trinity_gfx_dynamic_mgpg_enable(rdev, false);
 576	if (pi->enable_gfx_clock_gating)
 577		trinity_gfx_clockgating_enable(rdev, false);
 578	if (pi->enable_mg_clock_gating) {
 579		trinity_mg_clockgating_enable(rdev, false);
 580		trinity_ls_clockgating_enable(rdev, false);
 581	}
 582}
 583
 584static void trinity_set_divider_value(struct radeon_device *rdev,
 585				      u32 index, u32 sclk)
 586{
 587	struct atom_clock_dividers  dividers;
 588	int ret;
 589	u32 value;
 590	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 591
 592	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 593					     sclk, false, &dividers);
 594	if (ret)
 595		return;
 596
 597	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 598	value &= ~CLK_DIVIDER_MASK;
 599	value |= CLK_DIVIDER(dividers.post_div);
 600	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 601
 602	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 603					     sclk/2, false, &dividers);
 604	if (ret)
 605		return;
 606
 607	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
 608	value &= ~PD_SCLK_DIVIDER_MASK;
 609	value |= PD_SCLK_DIVIDER(dividers.post_div);
 610	WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
 611}
 612
 613static void trinity_set_ds_dividers(struct radeon_device *rdev,
 614				    u32 index, u32 divider)
 615{
 616	u32 value;
 617	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 618
 619	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 620	value &= ~DS_DIV_MASK;
 621	value |= DS_DIV(divider);
 622	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 623}
 624
 625static void trinity_set_ss_dividers(struct radeon_device *rdev,
 626				    u32 index, u32 divider)
 627{
 628	u32 value;
 629	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 630
 631	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 632	value &= ~DS_SH_DIV_MASK;
 633	value |= DS_SH_DIV(divider);
 634	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 635}
 636
 637static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
 638{
 639	struct trinity_power_info *pi = trinity_get_pi(rdev);
 640	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
 641	u32 value;
 642	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 643
 644	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 645	value &= ~VID_MASK;
 646	value |= VID(vid_7bit);
 647	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 648
 649	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 650	value &= ~LVRT_MASK;
 651	value |= LVRT(0);
 652	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 653}
 654
 655static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
 656				       u32 index, u32 gnb_slow)
 657{
 658	u32 value;
 659	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 660
 661	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 662	value &= ~GNB_SLOW_MASK;
 663	value |= GNB_SLOW(gnb_slow);
 664	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 665}
 666
 667static void trinity_set_force_nbp_state(struct radeon_device *rdev,
 668					u32 index, u32 force_nbp_state)
 669{
 670	u32 value;
 671	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 672
 673	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 674	value &= ~FORCE_NBPS1_MASK;
 675	value |= FORCE_NBPS1(force_nbp_state);
 676	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 677}
 678
 679static void trinity_set_display_wm(struct radeon_device *rdev,
 680				   u32 index, u32 wm)
 681{
 682	u32 value;
 683	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 684
 685	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 686	value &= ~DISPLAY_WM_MASK;
 687	value |= DISPLAY_WM(wm);
 688	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 689}
 690
 691static void trinity_set_vce_wm(struct radeon_device *rdev,
 692			       u32 index, u32 wm)
 693{
 694	u32 value;
 695	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 696
 697	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 698	value &= ~VCE_WM_MASK;
 699	value |= VCE_WM(wm);
 700	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 701}
 702
 703static void trinity_set_at(struct radeon_device *rdev,
 704			   u32 index, u32 at)
 705{
 706	u32 value;
 707	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 708
 709	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
 710	value &= ~AT_MASK;
 711	value |= AT(at);
 712	WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
 713}
 714
 715static void trinity_program_power_level(struct radeon_device *rdev,
 716					struct trinity_pl *pl, u32 index)
 717{
 718	struct trinity_power_info *pi = trinity_get_pi(rdev);
 719
 720	if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
 721		return;
 722
 723	trinity_set_divider_value(rdev, index, pl->sclk);
 724	trinity_set_vid(rdev, index, pl->vddc_index);
 725	trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
 726	trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
 727	trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
 728	trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
 729	trinity_set_display_wm(rdev, index, pl->display_wm);
 730	trinity_set_vce_wm(rdev, index, pl->vce_wm);
 731	trinity_set_at(rdev, index, pi->at[index]);
 732}
 733
 734static void trinity_power_level_enable_disable(struct radeon_device *rdev,
 735					       u32 index, bool enable)
 736{
 737	u32 value;
 738	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 739
 740	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 741	value &= ~STATE_VALID_MASK;
 742	if (enable)
 743		value |= STATE_VALID(1);
 744	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 745}
 746
 747static bool trinity_dpm_enabled(struct radeon_device *rdev)
 748{
 749	if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
 750		return true;
 751	else
 752		return false;
 753}
 754
 755static void trinity_start_dpm(struct radeon_device *rdev)
 756{
 757	u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 758
 759	value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
 760	value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
 761	WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
 762
 763	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 764	WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
 765
 766	trinity_dpm_config(rdev, true);
 767}
 768
 769static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
 770{
 771	int i;
 772
 773	for (i = 0; i < rdev->usec_timeout; i++) {
 774		if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
 775			break;
 776		udelay(1);
 777	}
 778	for (i = 0; i < rdev->usec_timeout; i++) {
 779		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
 780			break;
 781		udelay(1);
 782	}
 783	for (i = 0; i < rdev->usec_timeout; i++) {
 784		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 785			break;
 786		udelay(1);
 787	}
 788}
 789
 790static void trinity_stop_dpm(struct radeon_device *rdev)
 791{
 792	u32 sclk_dpm_cntl;
 793
 794	WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
 795
 796	sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 797	sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
 798	WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
 799
 800	trinity_dpm_config(rdev, false);
 801}
 802
 803static void trinity_start_am(struct radeon_device *rdev)
 804{
 805	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 806}
 807
 808static void trinity_reset_am(struct radeon_device *rdev)
 809{
 810	WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
 811		 ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 812}
 813
 814static void trinity_wait_for_level_0(struct radeon_device *rdev)
 815{
 816	int i;
 817
 818	for (i = 0; i < rdev->usec_timeout; i++) {
 819		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 820			break;
 821		udelay(1);
 822	}
 823}
 824
 825static void trinity_enable_power_level_0(struct radeon_device *rdev)
 826{
 827	trinity_power_level_enable_disable(rdev, 0, true);
 828}
 829
 830static void trinity_force_level_0(struct radeon_device *rdev)
 831{
 832	trinity_dpm_force_state(rdev, 0);
 833}
 834
 835static void trinity_unforce_levels(struct radeon_device *rdev)
 836{
 837	trinity_dpm_no_forced_level(rdev);
 838}
 839
 840static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
 841						struct radeon_ps *new_rps,
 842						struct radeon_ps *old_rps)
 843{
 844	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 845	struct trinity_ps *old_ps = trinity_get_ps(old_rps);
 846	u32 i;
 847	u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
 848
 849	for (i = 0; i < new_ps->num_levels; i++) {
 850		trinity_program_power_level(rdev, &new_ps->levels[i], i);
 851		trinity_power_level_enable_disable(rdev, i, true);
 852	}
 853
 854	for (i = new_ps->num_levels; i < n_current_state_levels; i++)
 855		trinity_power_level_enable_disable(rdev, i, false);
 856}
 857
 858static void trinity_program_bootup_state(struct radeon_device *rdev)
 859{
 860	struct trinity_power_info *pi = trinity_get_pi(rdev);
 861	u32 i;
 862
 863	trinity_program_power_level(rdev, &pi->boot_pl, 0);
 864	trinity_power_level_enable_disable(rdev, 0, true);
 865
 866	for (i = 1; i < 8; i++)
 867		trinity_power_level_enable_disable(rdev, i, false);
 868}
 869
 870static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
 871					  struct radeon_ps *rps)
 872{
 873	struct trinity_ps *ps = trinity_get_ps(rps);
 874	u32 uvdstates = (ps->vclk_low_divider |
 875			 ps->vclk_high_divider << 8 |
 876			 ps->dclk_low_divider << 16 |
 877			 ps->dclk_high_divider << 24);
 878
 879	WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
 880}
 881
 882static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
 883					   u32 interval)
 884{
 885	u32 p, u;
 886	u32 tp = RREG32_SMC(PM_TP);
 887	u32 val;
 888	u32 xclk = radeon_get_xclk(rdev);
 889
 890	r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
 891
 892	val = (p + tp - 1) / tp;
 893
 894	WREG32_SMC(SMU_UVD_DPM_CNTL, val);
 895}
 896
 897static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
 898{
 899	if ((rps->vclk == 0) && (rps->dclk == 0))
 900		return true;
 901	else
 902		return false;
 903}
 904
 905static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
 906				     struct radeon_ps *rps2)
 907{
 908	struct trinity_ps *ps1 = trinity_get_ps(rps1);
 909	struct trinity_ps *ps2 = trinity_get_ps(rps2);
 910
 911	if ((rps1->vclk == rps2->vclk) &&
 912	    (rps1->dclk == rps2->dclk) &&
 913	    (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
 914	    (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
 915	    (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
 916	    (ps1->dclk_high_divider == ps2->dclk_high_divider))
 917		return true;
 918	else
 919		return false;
 920}
 921
 922static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
 923				     struct radeon_ps *new_rps,
 924				     struct radeon_ps *old_rps)
 925{
 926	struct trinity_power_info *pi = trinity_get_pi(rdev);
 927
 928	if (pi->enable_gfx_power_gating) {
 929		trinity_gfx_powergating_enable(rdev, false);
 930	}
 931
 932	if (pi->uvd_dpm) {
 933		if (trinity_uvd_clocks_zero(new_rps) &&
 934		    !trinity_uvd_clocks_zero(old_rps)) {
 935			trinity_setup_uvd_dpm_interval(rdev, 0);
 936		} else if (!trinity_uvd_clocks_zero(new_rps)) {
 937			trinity_setup_uvd_clock_table(rdev, new_rps);
 938
 939			if (trinity_uvd_clocks_zero(old_rps)) {
 940				u32 tmp = RREG32(CG_MISC_REG);
 941				tmp &= 0xfffffffd;
 942				WREG32(CG_MISC_REG, tmp);
 943
 944				radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 945
 946				trinity_setup_uvd_dpm_interval(rdev, 3000);
 947			}
 948		}
 949		trinity_uvd_dpm_config(rdev);
 950	} else {
 951		if (trinity_uvd_clocks_zero(new_rps) ||
 952		    trinity_uvd_clocks_equal(new_rps, old_rps))
 953			return;
 954
 955		radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 956	}
 957
 958	if (pi->enable_gfx_power_gating) {
 959		trinity_gfx_powergating_enable(rdev, true);
 960	}
 961}
 962
 963static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
 964						       struct radeon_ps *new_rps,
 965						       struct radeon_ps *old_rps)
 966{
 967	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 968	struct trinity_ps *current_ps = trinity_get_ps(new_rps);
 969
 970	if (new_ps->levels[new_ps->num_levels - 1].sclk >=
 971	    current_ps->levels[current_ps->num_levels - 1].sclk)
 972		return;
 973
 974	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 975}
 976
 977static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
 978						      struct radeon_ps *new_rps,
 979						      struct radeon_ps *old_rps)
 980{
 981	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 982	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
 983
 984	if (new_ps->levels[new_ps->num_levels - 1].sclk <
 985	    current_ps->levels[current_ps->num_levels - 1].sclk)
 986		return;
 987
 988	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 989}
 990
 991static void trinity_set_vce_clock(struct radeon_device *rdev,
 992				  struct radeon_ps *new_rps,
 993				  struct radeon_ps *old_rps)
 994{
 995	if ((old_rps->evclk != new_rps->evclk) ||
 996	    (old_rps->ecclk != new_rps->ecclk)) {
 997		/* turn the clocks on when encoding, off otherwise */
 998		if (new_rps->evclk || new_rps->ecclk)
 999			vce_v1_0_enable_mgcg(rdev, false);
1000		else
1001			vce_v1_0_enable_mgcg(rdev, true);
1002		radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
1003	}
1004}
1005
1006static void trinity_program_ttt(struct radeon_device *rdev)
1007{
1008	struct trinity_power_info *pi = trinity_get_pi(rdev);
1009	u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
1010
1011	value &= ~(HT_MASK | LT_MASK);
1012	value |= HT((pi->thermal_auto_throttling + 49) * 8);
1013	value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
1014	WREG32_SMC(SMU_SCLK_DPM_TTT, value);
1015}
1016
1017static void trinity_enable_att(struct radeon_device *rdev)
1018{
1019	u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
1020
1021	value &= ~SCLK_TT_EN_MASK;
1022	value |= SCLK_TT_EN(1);
1023	WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
1024}
1025
1026static void trinity_program_sclk_dpm(struct radeon_device *rdev)
1027{
1028	u32 p, u;
1029	u32 tp = RREG32_SMC(PM_TP);
1030	u32 ni;
1031	u32 xclk = radeon_get_xclk(rdev);
1032	u32 value;
1033
1034	r600_calculate_u_and_p(400, xclk, 16, &p, &u);
1035
1036	ni = (p + tp - 1) / tp;
1037
1038	value = RREG32_SMC(PM_I_CNTL_1);
1039	value &= ~SCLK_DPM_MASK;
1040	value |= SCLK_DPM(ni);
1041	WREG32_SMC(PM_I_CNTL_1, value);
1042}
1043
1044static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
1045						 int min_temp, int max_temp)
1046{
1047	int low_temp = 0 * 1000;
1048	int high_temp = 255 * 1000;
1049
1050	if (low_temp < min_temp)
1051		low_temp = min_temp;
1052	if (high_temp > max_temp)
1053		high_temp = max_temp;
1054	if (high_temp < low_temp) {
1055		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1056		return -EINVAL;
1057	}
1058
1059	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1060	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1061
1062	rdev->pm.dpm.thermal.min_temp = low_temp;
1063	rdev->pm.dpm.thermal.max_temp = high_temp;
1064
1065	return 0;
1066}
1067
1068static void trinity_update_current_ps(struct radeon_device *rdev,
1069				      struct radeon_ps *rps)
1070{
1071	struct trinity_ps *new_ps = trinity_get_ps(rps);
1072	struct trinity_power_info *pi = trinity_get_pi(rdev);
1073
1074	pi->current_rps = *rps;
1075	pi->current_ps = *new_ps;
1076	pi->current_rps.ps_priv = &pi->current_ps;
1077}
1078
1079static void trinity_update_requested_ps(struct radeon_device *rdev,
1080					struct radeon_ps *rps)
1081{
1082	struct trinity_ps *new_ps = trinity_get_ps(rps);
1083	struct trinity_power_info *pi = trinity_get_pi(rdev);
1084
1085	pi->requested_rps = *rps;
1086	pi->requested_ps = *new_ps;
1087	pi->requested_rps.ps_priv = &pi->requested_ps;
1088}
1089
1090void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1091{
1092	struct trinity_power_info *pi = trinity_get_pi(rdev);
1093
1094	if (pi->enable_bapm) {
1095		trinity_acquire_mutex(rdev);
1096		trinity_dpm_bapm_enable(rdev, enable);
1097		trinity_release_mutex(rdev);
1098	}
1099}
1100
1101int trinity_dpm_enable(struct radeon_device *rdev)
1102{
1103	struct trinity_power_info *pi = trinity_get_pi(rdev);
1104
1105	trinity_acquire_mutex(rdev);
1106
1107	if (trinity_dpm_enabled(rdev)) {
1108		trinity_release_mutex(rdev);
1109		return -EINVAL;
1110	}
1111
1112	trinity_program_bootup_state(rdev);
1113	sumo_program_vc(rdev, 0x00C00033);
1114	trinity_start_am(rdev);
1115	if (pi->enable_auto_thermal_throttling) {
1116		trinity_program_ttt(rdev);
1117		trinity_enable_att(rdev);
1118	}
1119	trinity_program_sclk_dpm(rdev);
1120	trinity_start_dpm(rdev);
1121	trinity_wait_for_dpm_enabled(rdev);
1122	trinity_dpm_bapm_enable(rdev, false);
1123	trinity_release_mutex(rdev);
1124
1125	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1126
1127	return 0;
1128}
1129
1130int trinity_dpm_late_enable(struct radeon_device *rdev)
1131{
1132	int ret;
1133
1134	trinity_acquire_mutex(rdev);
1135	trinity_enable_clock_power_gating(rdev);
1136
1137	if (rdev->irq.installed &&
1138	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1139		ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1140		if (ret) {
1141			trinity_release_mutex(rdev);
1142			return ret;
1143		}
1144		rdev->irq.dpm_thermal = true;
1145		radeon_irq_set(rdev);
1146	}
1147	trinity_release_mutex(rdev);
1148
1149	return 0;
1150}
1151
1152void trinity_dpm_disable(struct radeon_device *rdev)
1153{
1154	trinity_acquire_mutex(rdev);
1155	if (!trinity_dpm_enabled(rdev)) {
1156		trinity_release_mutex(rdev);
1157		return;
1158	}
1159	trinity_dpm_bapm_enable(rdev, false);
1160	trinity_disable_clock_power_gating(rdev);
1161	sumo_clear_vc(rdev);
1162	trinity_wait_for_level_0(rdev);
1163	trinity_stop_dpm(rdev);
1164	trinity_reset_am(rdev);
1165	trinity_release_mutex(rdev);
1166
1167	if (rdev->irq.installed &&
1168	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1169		rdev->irq.dpm_thermal = false;
1170		radeon_irq_set(rdev);
1171	}
1172
1173	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1174}
1175
1176static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1177{
1178	struct trinity_power_info *pi = trinity_get_pi(rdev);
1179
1180	pi->min_sclk_did =
1181		(RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1182}
1183
1184static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1185				  struct radeon_ps *rps)
1186{
1187	struct trinity_power_info *pi = trinity_get_pi(rdev);
1188	struct trinity_ps *new_ps = trinity_get_ps(rps);
1189	u32 nbpsconfig;
1190
1191	if (pi->sys_info.nb_dpm_enable) {
1192		nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1193		nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1194		nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1195			       Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1196			       DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1197			       DpmXNbPsHi(new_ps->DpmXNbPsHi));
1198		WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1199	}
1200}
1201
1202int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1203					enum radeon_dpm_forced_level level)
1204{
1205	struct trinity_power_info *pi = trinity_get_pi(rdev);
1206	struct radeon_ps *rps = &pi->current_rps;
1207	struct trinity_ps *ps = trinity_get_ps(rps);
1208	int i, ret;
1209
1210	if (ps->num_levels <= 1)
1211		return 0;
1212
1213	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1214		/* not supported by the hw */
1215		return -EINVAL;
1216	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1217		ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1218		if (ret)
1219			return ret;
1220	} else {
1221		for (i = 0; i < ps->num_levels; i++) {
1222			ret = trinity_dpm_n_levels_disabled(rdev, 0);
1223			if (ret)
1224				return ret;
1225		}
1226	}
1227
1228	rdev->pm.dpm.forced_level = level;
1229
1230	return 0;
1231}
1232
1233int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1234{
1235	struct trinity_power_info *pi = trinity_get_pi(rdev);
1236	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1237	struct radeon_ps *new_ps = &requested_ps;
1238
1239	trinity_update_requested_ps(rdev, new_ps);
1240
1241	trinity_apply_state_adjust_rules(rdev,
1242					 &pi->requested_rps,
1243					 &pi->current_rps);
1244
1245	return 0;
1246}
1247
1248int trinity_dpm_set_power_state(struct radeon_device *rdev)
1249{
1250	struct trinity_power_info *pi = trinity_get_pi(rdev);
1251	struct radeon_ps *new_ps = &pi->requested_rps;
1252	struct radeon_ps *old_ps = &pi->current_rps;
1253
1254	trinity_acquire_mutex(rdev);
1255	if (pi->enable_dpm) {
1256		if (pi->enable_bapm)
1257			trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1258		trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1259		trinity_enable_power_level_0(rdev);
1260		trinity_force_level_0(rdev);
1261		trinity_wait_for_level_0(rdev);
1262		trinity_setup_nbp_sim(rdev, new_ps);
1263		trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1264		trinity_force_level_0(rdev);
1265		trinity_unforce_levels(rdev);
1266		trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1267		trinity_set_vce_clock(rdev, new_ps, old_ps);
1268	}
1269	trinity_release_mutex(rdev);
1270
1271	return 0;
1272}
1273
1274void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1275{
1276	struct trinity_power_info *pi = trinity_get_pi(rdev);
1277	struct radeon_ps *new_ps = &pi->requested_rps;
1278
1279	trinity_update_current_ps(rdev, new_ps);
1280}
1281
1282void trinity_dpm_setup_asic(struct radeon_device *rdev)
1283{
1284	trinity_acquire_mutex(rdev);
1285	sumo_program_sstp(rdev);
1286	sumo_take_smu_control(rdev, true);
1287	trinity_get_min_sclk_divider(rdev);
1288	trinity_release_mutex(rdev);
1289}
1290
1291#if 0
1292void trinity_dpm_reset_asic(struct radeon_device *rdev)
1293{
1294	struct trinity_power_info *pi = trinity_get_pi(rdev);
1295
1296	trinity_acquire_mutex(rdev);
1297	if (pi->enable_dpm) {
1298		trinity_enable_power_level_0(rdev);
1299		trinity_force_level_0(rdev);
1300		trinity_wait_for_level_0(rdev);
1301		trinity_program_bootup_state(rdev);
1302		trinity_force_level_0(rdev);
1303		trinity_unforce_levels(rdev);
1304	}
1305	trinity_release_mutex(rdev);
1306}
1307#endif
1308
1309static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1310						  u32 vid_2bit)
1311{
1312	struct trinity_power_info *pi = trinity_get_pi(rdev);
1313	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1314	u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1315	u32 step = (svi_mode == 0) ? 1250 : 625;
1316	u32 delta = vid_7bit * step + 50;
1317
1318	if (delta > 155000)
1319		return 0;
1320
1321	return (155000 - delta) / 100;
1322}
1323
1324static void trinity_patch_boot_state(struct radeon_device *rdev,
1325				     struct trinity_ps *ps)
1326{
1327	struct trinity_power_info *pi = trinity_get_pi(rdev);
1328
1329	ps->num_levels = 1;
1330	ps->nbps_flags = 0;
1331	ps->bapm_flags = 0;
1332	ps->levels[0] = pi->boot_pl;
1333}
1334
1335static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1336{
1337	if (sclk < 20000)
1338		return 1;
1339	return 0;
1340}
1341
1342static void trinity_construct_boot_state(struct radeon_device *rdev)
1343{
1344	struct trinity_power_info *pi = trinity_get_pi(rdev);
1345
1346	pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1347	pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1348	pi->boot_pl.ds_divider_index = 0;
1349	pi->boot_pl.ss_divider_index = 0;
1350	pi->boot_pl.allow_gnb_slow = 1;
1351	pi->boot_pl.force_nbp_state = 0;
1352	pi->boot_pl.display_wm = 0;
1353	pi->boot_pl.vce_wm = 0;
1354	pi->current_ps.num_levels = 1;
1355	pi->current_ps.levels[0] = pi->boot_pl;
1356}
1357
1358static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1359						  u32 sclk, u32 min_sclk_in_sr)
1360{
1361	struct trinity_power_info *pi = trinity_get_pi(rdev);
1362	u32 i;
1363	u32 temp;
1364	u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1365		min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1366
1367	if (sclk < min)
1368		return 0;
1369
1370	if (!pi->enable_sclk_ds)
1371		return 0;
1372
1373	for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1374		temp = sclk / sumo_get_sleep_divider_from_id(i);
1375		if (temp >= min || i == 0)
1376			break;
1377	}
1378
1379	return (u8)i;
1380}
1381
1382static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1383					  u32 lower_limit)
1384{
1385	struct trinity_power_info *pi = trinity_get_pi(rdev);
1386	u32 i;
1387
1388	for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1389		if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1390			return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1391	}
1392
1393	if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1394		DRM_ERROR("engine clock out of range!");
1395
1396	return 0;
1397}
1398
1399static void trinity_patch_thermal_state(struct radeon_device *rdev,
1400					struct trinity_ps *ps,
1401					struct trinity_ps *current_ps)
1402{
1403	struct trinity_power_info *pi = trinity_get_pi(rdev);
1404	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1405	u32 current_vddc;
1406	u32 current_sclk;
1407	u32 current_index = 0;
1408
1409	if (current_ps) {
1410		current_vddc = current_ps->levels[current_index].vddc_index;
1411		current_sclk = current_ps->levels[current_index].sclk;
1412	} else {
1413		current_vddc = pi->boot_pl.vddc_index;
1414		current_sclk = pi->boot_pl.sclk;
1415	}
1416
1417	ps->levels[0].vddc_index = current_vddc;
1418
1419	if (ps->levels[0].sclk > current_sclk)
1420		ps->levels[0].sclk = current_sclk;
1421
1422	ps->levels[0].ds_divider_index =
1423		trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1424	ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1425	ps->levels[0].allow_gnb_slow = 1;
1426	ps->levels[0].force_nbp_state = 0;
1427	ps->levels[0].display_wm = 0;
1428	ps->levels[0].vce_wm =
1429		trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1430}
1431
1432static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1433				       struct trinity_ps *ps, u32 index)
1434{
1435	if (ps == NULL || ps->num_levels <= 1)
1436		return 0;
1437	else if (ps->num_levels == 2) {
1438		if (index == 0)
1439			return 0;
1440		else
1441			return 1;
1442	} else {
1443		if (index == 0)
1444			return 0;
1445		else if (ps->levels[index].sclk < 30000)
1446			return 0;
1447		else
1448			return 1;
1449	}
1450}
1451
1452static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1453				       struct radeon_ps *rps)
1454{
1455	struct trinity_power_info *pi = trinity_get_pi(rdev);
1456	u32 i = 0;
1457
1458	for (i = 0; i < 4; i++) {
1459		if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1460		    (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1461		    break;
1462	}
1463
1464	if (i >= 4) {
1465		DRM_ERROR("UVD clock index not found!\n");
1466		i = 3;
1467	}
1468	return i;
1469}
1470
1471static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1472				     struct radeon_ps *rps)
1473{
1474	struct trinity_ps *ps = trinity_get_ps(rps);
1475	struct trinity_power_info *pi = trinity_get_pi(rdev);
1476	u32 high_index = 0;
1477	u32 low_index = 0;
1478
1479	if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1480		high_index = trinity_get_uvd_clock_index(rdev, rps);
1481
1482		switch(high_index) {
1483		case 3:
1484		case 2:
1485			low_index = 1;
1486			break;
1487		case 1:
1488		case 0:
1489		default:
1490			low_index = 0;
1491			break;
1492		}
1493
1494		ps->vclk_low_divider =
1495			pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1496		ps->dclk_low_divider =
1497			pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1498		ps->vclk_high_divider =
1499			pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1500		ps->dclk_high_divider =
1501			pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1502	}
1503}
1504
1505static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1506					 u32 evclk, u32 ecclk, u16 *voltage)
1507{
1508	u32 i;
1509	int ret = -EINVAL;
1510	struct radeon_vce_clock_voltage_dependency_table *table =
1511		&rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1512
1513	if (((evclk == 0) && (ecclk == 0)) ||
1514	    (table && (table->count == 0))) {
1515		*voltage = 0;
1516		return 0;
1517	}
1518
1519	for (i = 0; i < table->count; i++) {
1520		if ((evclk <= table->entries[i].evclk) &&
1521		    (ecclk <= table->entries[i].ecclk)) {
1522			*voltage = table->entries[i].v;
1523			ret = 0;
1524			break;
1525		}
1526	}
1527
1528	/* if no match return the highest voltage */
1529	if (ret)
1530		*voltage = table->entries[table->count - 1].v;
1531
1532	return ret;
1533}
1534
1535static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1536					     struct radeon_ps *new_rps,
1537					     struct radeon_ps *old_rps)
1538{
1539	struct trinity_ps *ps = trinity_get_ps(new_rps);
1540	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1541	struct trinity_power_info *pi = trinity_get_pi(rdev);
1542	u32 min_voltage = 0; /* ??? */
1543	u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1544	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1545	u32 i;
1546	u16 min_vce_voltage;
1547	bool force_high;
1548	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1549
1550	if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1551		return trinity_patch_thermal_state(rdev, ps, current_ps);
1552
1553	trinity_adjust_uvd_state(rdev, new_rps);
1554
1555	if (new_rps->vce_active) {
1556		new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1557		new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1558	} else {
1559		new_rps->evclk = 0;
1560		new_rps->ecclk = 0;
1561	}
1562
1563	for (i = 0; i < ps->num_levels; i++) {
1564		if (ps->levels[i].vddc_index < min_voltage)
1565			ps->levels[i].vddc_index = min_voltage;
1566
1567		if (ps->levels[i].sclk < min_sclk)
1568			ps->levels[i].sclk =
1569				trinity_get_valid_engine_clock(rdev, min_sclk);
1570
1571		/* patch in vce limits */
1572		if (new_rps->vce_active) {
1573			/* sclk */
1574			if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1575				ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1576			/* vddc */
1577			trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1578			if (ps->levels[i].vddc_index < min_vce_voltage)
1579				ps->levels[i].vddc_index = min_vce_voltage;
1580		}
1581
1582		ps->levels[i].ds_divider_index =
1583			sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1584
1585		ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1586
1587		ps->levels[i].allow_gnb_slow = 1;
1588		ps->levels[i].force_nbp_state = 0;
1589		ps->levels[i].display_wm =
1590			trinity_calculate_display_wm(rdev, ps, i);
1591		ps->levels[i].vce_wm =
1592			trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1593	}
1594
1595	if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1596	    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1597		ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1598
1599	if (pi->sys_info.nb_dpm_enable) {
1600		ps->Dpm0PgNbPsLo = 0x1;
1601		ps->Dpm0PgNbPsHi = 0x0;
1602		ps->DpmXNbPsLo = 0x2;
1603		ps->DpmXNbPsHi = 0x1;
1604
1605		if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1606		    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1607			force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1608				      ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1609				       (pi->sys_info.uma_channel_number == 1)));
1610			force_high = (num_active_displays >= 3) || force_high;
1611			ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1612			ps->Dpm0PgNbPsHi = 0x1;
1613			ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1614			ps->DpmXNbPsHi = 0x2;
1615			ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1616		}
1617	}
1618}
1619
1620static void trinity_cleanup_asic(struct radeon_device *rdev)
1621{
1622	sumo_take_smu_control(rdev, false);
1623}
1624
1625#if 0
1626static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1627{
1628	struct trinity_power_info *pi = trinity_get_pi(rdev);
1629
1630	if (pi->voltage_drop_in_dce)
1631		trinity_dce_enable_voltage_adjustment(rdev, false);
1632}
1633#endif
1634
1635static void trinity_add_dccac_value(struct radeon_device *rdev)
1636{
1637	u32 gpu_cac_avrg_cntl_window_size;
1638	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1639	u64 disp_clk = rdev->clock.default_dispclk / 100;
1640	u32 dc_cac_value;
1641
1642	gpu_cac_avrg_cntl_window_size =
1643		(RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1644
1645	dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1646			     (32 - gpu_cac_avrg_cntl_window_size));
1647
1648	WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1649}
1650
1651void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1652{
1653	struct trinity_power_info *pi = trinity_get_pi(rdev);
1654
1655	if (pi->voltage_drop_in_dce)
1656		trinity_dce_enable_voltage_adjustment(rdev, true);
1657	trinity_add_dccac_value(rdev);
1658}
1659
1660union power_info {
1661	struct _ATOM_POWERPLAY_INFO info;
1662	struct _ATOM_POWERPLAY_INFO_V2 info_2;
1663	struct _ATOM_POWERPLAY_INFO_V3 info_3;
1664	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1665	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1666	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1667};
1668
1669union pplib_clock_info {
1670	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1671	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1672	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1673	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1674};
1675
1676union pplib_power_state {
1677	struct _ATOM_PPLIB_STATE v1;
1678	struct _ATOM_PPLIB_STATE_V2 v2;
1679};
1680
1681static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1682					       struct radeon_ps *rps,
1683					       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1684					       u8 table_rev)
1685{
1686	struct trinity_ps *ps = trinity_get_ps(rps);
1687
1688	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1689	rps->class = le16_to_cpu(non_clock_info->usClassification);
1690	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1691
1692	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1693		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1694		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1695	} else {
1696		rps->vclk = 0;
1697		rps->dclk = 0;
1698	}
1699
1700	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1701		rdev->pm.dpm.boot_ps = rps;
1702		trinity_patch_boot_state(rdev, ps);
1703	}
1704	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1705		rdev->pm.dpm.uvd_ps = rps;
1706}
1707
1708static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1709					   struct radeon_ps *rps, int index,
1710					   union pplib_clock_info *clock_info)
1711{
1712	struct trinity_power_info *pi = trinity_get_pi(rdev);
1713	struct trinity_ps *ps = trinity_get_ps(rps);
1714	struct trinity_pl *pl = &ps->levels[index];
1715	u32 sclk;
1716
1717	sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1718	sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1719	pl->sclk = sclk;
1720	pl->vddc_index = clock_info->sumo.vddcIndex;
1721
1722	ps->num_levels = index + 1;
1723
1724	if (pi->enable_sclk_ds) {
1725		pl->ds_divider_index = 5;
1726		pl->ss_divider_index = 5;
1727	}
1728}
1729
1730static int trinity_parse_power_table(struct radeon_device *rdev)
1731{
1732	struct radeon_mode_info *mode_info = &rdev->mode_info;
1733	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1734	union pplib_power_state *power_state;
1735	int i, j, k, non_clock_array_index, clock_array_index;
1736	union pplib_clock_info *clock_info;
1737	struct _StateArray *state_array;
1738	struct _ClockInfoArray *clock_info_array;
1739	struct _NonClockInfoArray *non_clock_info_array;
1740	union power_info *power_info;
1741	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1742	u16 data_offset;
1743	u8 frev, crev;
1744	u8 *power_state_offset;
1745	struct sumo_ps *ps;
1746
1747	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1748				   &frev, &crev, &data_offset))
1749		return -EINVAL;
1750	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1751
1752	state_array = (struct _StateArray *)
1753		(mode_info->atom_context->bios + data_offset +
1754		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
1755	clock_info_array = (struct _ClockInfoArray *)
1756		(mode_info->atom_context->bios + data_offset +
1757		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1758	non_clock_info_array = (struct _NonClockInfoArray *)
1759		(mode_info->atom_context->bios + data_offset +
1760		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1761
1762	rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1763				  sizeof(struct radeon_ps),
1764				  GFP_KERNEL);
1765	if (!rdev->pm.dpm.ps)
1766		return -ENOMEM;
1767	power_state_offset = (u8 *)state_array->states;
1768	for (i = 0; i < state_array->ucNumEntries; i++) {
1769		u8 *idx;
1770		power_state = (union pplib_power_state *)power_state_offset;
1771		non_clock_array_index = power_state->v2.nonClockInfoIndex;
1772		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1773			&non_clock_info_array->nonClockInfo[non_clock_array_index];
1774		if (!rdev->pm.power_state[i].clock_info)
 
1775			return -EINVAL;
 
1776		ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1777		if (ps == NULL) {
1778			kfree(rdev->pm.dpm.ps);
1779			return -ENOMEM;
1780		}
1781		rdev->pm.dpm.ps[i].ps_priv = ps;
1782		k = 0;
1783		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1784		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1785			clock_array_index = idx[j];
1786			if (clock_array_index >= clock_info_array->ucNumEntries)
1787				continue;
1788			if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1789				break;
1790			clock_info = (union pplib_clock_info *)
1791				((u8 *)&clock_info_array->clockInfo[0] +
1792				 (clock_array_index * clock_info_array->ucEntrySize));
1793			trinity_parse_pplib_clock_info(rdev,
1794						       &rdev->pm.dpm.ps[i], k,
1795						       clock_info);
1796			k++;
1797		}
1798		trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1799						   non_clock_info,
1800						   non_clock_info_array->ucEntrySize);
1801		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1802	}
1803	rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1804
1805	/* fill in the vce power states */
1806	for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1807		u32 sclk;
1808		clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1809		clock_info = (union pplib_clock_info *)
1810			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1811		sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1812		sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1813		rdev->pm.dpm.vce_states[i].sclk = sclk;
1814		rdev->pm.dpm.vce_states[i].mclk = 0;
1815	}
1816
1817	return 0;
1818}
1819
1820union igp_info {
1821	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1822	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1823	struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1824	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1825	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1826};
1827
1828static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1829{
1830	struct trinity_power_info *pi = trinity_get_pi(rdev);
1831	u32 divider;
1832
1833	if (did >= 8 && did <= 0x3f)
1834		divider = did * 25;
1835	else if (did > 0x3f && did <= 0x5f)
1836		divider = (did - 64) * 50 + 1600;
1837	else if (did > 0x5f && did <= 0x7e)
1838		divider = (did - 96) * 100 + 3200;
1839	else if (did == 0x7f)
1840		divider = 128 * 100;
1841	else
1842		return 10000;
1843
1844	return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1845}
1846
1847static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1848{
1849	struct trinity_power_info *pi = trinity_get_pi(rdev);
1850	struct radeon_mode_info *mode_info = &rdev->mode_info;
1851	int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1852	union igp_info *igp_info;
1853	u8 frev, crev;
1854	u16 data_offset;
1855	int i;
1856
1857	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1858				   &frev, &crev, &data_offset)) {
1859		igp_info = (union igp_info *)(mode_info->atom_context->bios +
1860					      data_offset);
1861
1862		if (crev != 7) {
1863			DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1864			return -EINVAL;
1865		}
1866		pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1867		pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1868		pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1869		pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1870		pi->sys_info.bootup_nb_voltage_index =
1871			le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1872		if (igp_info->info_7.ucHtcTmpLmt == 0)
1873			pi->sys_info.htc_tmp_lmt = 203;
1874		else
1875			pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1876		if (igp_info->info_7.ucHtcHystLmt == 0)
1877			pi->sys_info.htc_hyst_lmt = 5;
1878		else
1879			pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1880		if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1881			DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1882		}
1883
1884		if (pi->enable_nbps_policy)
1885			pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1886		else
1887			pi->sys_info.nb_dpm_enable = 0;
1888
1889		for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1890			pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1891			pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1892		}
1893
1894		pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1895		pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1896		pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1897		pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1898
1899		if (!pi->sys_info.nb_dpm_enable) {
1900			for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1901				pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1902				pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1903				pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1904			}
1905		}
1906
1907		pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1908
1909		sumo_construct_sclk_voltage_mapping_table(rdev,
1910							  &pi->sys_info.sclk_voltage_mapping_table,
1911							  igp_info->info_7.sAvail_SCLK);
1912		sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1913						 igp_info->info_7.sAvail_SCLK);
1914
1915		pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1916			igp_info->info_7.ucDPMState0VclkFid;
1917		pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1918			igp_info->info_7.ucDPMState1VclkFid;
1919		pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1920			igp_info->info_7.ucDPMState2VclkFid;
1921		pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1922			igp_info->info_7.ucDPMState3VclkFid;
1923
1924		pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1925			igp_info->info_7.ucDPMState0DclkFid;
1926		pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1927			igp_info->info_7.ucDPMState1DclkFid;
1928		pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1929			igp_info->info_7.ucDPMState2DclkFid;
1930		pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1931			igp_info->info_7.ucDPMState3DclkFid;
1932
1933		for (i = 0; i < 4; i++) {
1934			pi->sys_info.uvd_clock_table_entries[i].vclk =
1935				trinity_convert_did_to_freq(rdev,
1936							    pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1937			pi->sys_info.uvd_clock_table_entries[i].dclk =
1938				trinity_convert_did_to_freq(rdev,
1939							    pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1940		}
1941
1942
1943
1944	}
1945	return 0;
1946}
1947
1948int trinity_dpm_init(struct radeon_device *rdev)
1949{
1950	struct trinity_power_info *pi;
1951	int ret, i;
1952
1953	pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1954	if (pi == NULL)
1955		return -ENOMEM;
1956	rdev->pm.dpm.priv = pi;
1957
1958	for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1959		pi->at[i] = TRINITY_AT_DFLT;
1960
1961	if (radeon_bapm == -1) {
1962		/* There are stability issues reported on with
1963		 * bapm enabled when switching between AC and battery
1964		 * power.  At the same time, some MSI boards hang
1965		 * if it's not enabled and dpm is enabled.  Just enable
1966		 * it for MSI boards right now.
1967		 */
1968		if (rdev->pdev->subsystem_vendor == 0x1462)
1969			pi->enable_bapm = true;
1970		else
1971			pi->enable_bapm = false;
1972	} else if (radeon_bapm == 0) {
1973		pi->enable_bapm = false;
1974	} else {
1975		pi->enable_bapm = true;
1976	}
1977	pi->enable_nbps_policy = true;
1978	pi->enable_sclk_ds = true;
1979	pi->enable_gfx_power_gating = true;
1980	pi->enable_gfx_clock_gating = true;
1981	pi->enable_mg_clock_gating = false;
1982	pi->enable_gfx_dynamic_mgpg = false;
1983	pi->override_dynamic_mgpg = false;
1984	pi->enable_auto_thermal_throttling = true;
1985	pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1986	pi->uvd_dpm = true; /* ??? */
1987
1988	ret = trinity_parse_sys_info_table(rdev);
1989	if (ret)
1990		return ret;
1991
1992	trinity_construct_boot_state(rdev);
1993
1994	ret = r600_get_platform_caps(rdev);
1995	if (ret)
1996		return ret;
1997
1998	ret = r600_parse_extended_power_table(rdev);
1999	if (ret)
2000		return ret;
2001
2002	ret = trinity_parse_power_table(rdev);
2003	if (ret)
2004		return ret;
2005
2006	pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
2007	pi->enable_dpm = true;
2008
2009	return 0;
2010}
2011
2012void trinity_dpm_print_power_state(struct radeon_device *rdev,
2013				   struct radeon_ps *rps)
2014{
2015	int i;
2016	struct trinity_ps *ps = trinity_get_ps(rps);
2017
2018	r600_dpm_print_class_info(rps->class, rps->class2);
2019	r600_dpm_print_cap_info(rps->caps);
2020	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2021	for (i = 0; i < ps->num_levels; i++) {
2022		struct trinity_pl *pl = &ps->levels[i];
2023		printk("\t\tpower level %d    sclk: %u vddc: %u\n",
2024		       i, pl->sclk,
2025		       trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2026	}
2027	r600_dpm_print_ps_status(rdev, rps);
2028}
2029
2030void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
2031							 struct seq_file *m)
2032{
2033	struct trinity_power_info *pi = trinity_get_pi(rdev);
2034	struct radeon_ps *rps = &pi->current_rps;
2035	struct trinity_ps *ps = trinity_get_ps(rps);
2036	struct trinity_pl *pl;
2037	u32 current_index =
2038		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2039		CURRENT_STATE_SHIFT;
2040
2041	if (current_index >= ps->num_levels) {
2042		seq_printf(m, "invalid dpm profile %d\n", current_index);
2043	} else {
2044		pl = &ps->levels[current_index];
2045		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2046		seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2047			   current_index, pl->sclk,
2048			   trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2049	}
2050}
2051
2052u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2053{
2054	struct trinity_power_info *pi = trinity_get_pi(rdev);
2055	struct radeon_ps *rps = &pi->current_rps;
2056	struct trinity_ps *ps = trinity_get_ps(rps);
2057	struct trinity_pl *pl;
2058	u32 current_index =
2059		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2060		CURRENT_STATE_SHIFT;
2061
2062	if (current_index >= ps->num_levels) {
2063		return 0;
2064	} else {
2065		pl = &ps->levels[current_index];
2066		return pl->sclk;
2067	}
2068}
2069
2070u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2071{
2072	struct trinity_power_info *pi = trinity_get_pi(rdev);
2073
2074	return pi->sys_info.bootup_uma_clk;
2075}
2076
2077void trinity_dpm_fini(struct radeon_device *rdev)
2078{
2079	int i;
2080
2081	trinity_cleanup_asic(rdev); /* ??? */
2082
2083	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2084		kfree(rdev->pm.dpm.ps[i].ps_priv);
2085	}
2086	kfree(rdev->pm.dpm.ps);
2087	kfree(rdev->pm.dpm.priv);
2088	r600_free_extended_power_table(rdev);
2089}
2090
2091u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2092{
2093	struct trinity_power_info *pi = trinity_get_pi(rdev);
2094	struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2095
2096	if (low)
2097		return requested_state->levels[0].sclk;
2098	else
2099		return requested_state->levels[requested_state->num_levels - 1].sclk;
2100}
2101
2102u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2103{
2104	struct trinity_power_info *pi = trinity_get_pi(rdev);
2105
2106	return pi->sys_info.bootup_uma_clk;
2107}