1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include "drmP.h"
  28#include "amdgpu.h"
  29#include "amdgpu_atombios.h"
  30#include "amdgpu_ih.h"
  31#include "amdgpu_uvd.h"
  32#include "amdgpu_vce.h"
  33#include "cikd.h"
  34#include "atom.h"
  35#include "amd_pcie.h"
  36
  37#include "cik.h"
  38#include "gmc_v7_0.h"
  39#include "cik_ih.h"
  40#include "dce_v8_0.h"
  41#include "gfx_v7_0.h"
  42#include "cik_sdma.h"
  43#include "uvd_v4_2.h"
  44#include "vce_v2_0.h"
  45#include "cik_dpm.h"
  46
  47#include "uvd/uvd_4_2_d.h"
  48
  49#include "smu/smu_7_0_1_d.h"
  50#include "smu/smu_7_0_1_sh_mask.h"
  51
  52#include "dce/dce_8_0_d.h"
  53#include "dce/dce_8_0_sh_mask.h"
  54
  55#include "bif/bif_4_1_d.h"
  56#include "bif/bif_4_1_sh_mask.h"
  57
  58#include "gca/gfx_7_2_d.h"
  59#include "gca/gfx_7_2_enum.h"
  60#include "gca/gfx_7_2_sh_mask.h"
  61
  62#include "gmc/gmc_7_1_d.h"
  63#include "gmc/gmc_7_1_sh_mask.h"
  64
  65#include "oss/oss_2_0_d.h"
  66#include "oss/oss_2_0_sh_mask.h"
  67
  68#include "amdgpu_amdkfd.h"
  69#include "amdgpu_powerplay.h"
 
  70
  71/*
  72 * Indirect registers accessor
  73 */
  74static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  75{
  76	unsigned long flags;
  77	u32 r;
  78
  79	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  80	WREG32(mmPCIE_INDEX, reg);
  81	(void)RREG32(mmPCIE_INDEX);
  82	r = RREG32(mmPCIE_DATA);
  83	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  84	return r;
  85}
  86
  87static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  88{
  89	unsigned long flags;
  90
  91	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  92	WREG32(mmPCIE_INDEX, reg);
  93	(void)RREG32(mmPCIE_INDEX);
  94	WREG32(mmPCIE_DATA, v);
  95	(void)RREG32(mmPCIE_DATA);
  96	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  97}
  98
  99static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 100{
 101	unsigned long flags;
 102	u32 r;
 103
 104	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 105	WREG32(mmSMC_IND_INDEX_0, (reg));
 106	r = RREG32(mmSMC_IND_DATA_0);
 107	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 108	return r;
 109}
 110
 111static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 112{
 113	unsigned long flags;
 114
 115	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 116	WREG32(mmSMC_IND_INDEX_0, (reg));
 117	WREG32(mmSMC_IND_DATA_0, (v));
 118	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 119}
 120
 121static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 122{
 123	unsigned long flags;
 124	u32 r;
 125
 126	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 127	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 128	r = RREG32(mmUVD_CTX_DATA);
 129	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 130	return r;
 131}
 132
 133static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 134{
 135	unsigned long flags;
 136
 137	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 138	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 139	WREG32(mmUVD_CTX_DATA, (v));
 140	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 141}
 142
 143static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 144{
 145	unsigned long flags;
 146	u32 r;
 147
 148	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 149	WREG32(mmDIDT_IND_INDEX, (reg));
 150	r = RREG32(mmDIDT_IND_DATA);
 151	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 152	return r;
 153}
 154
 155static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 156{
 157	unsigned long flags;
 158
 159	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 160	WREG32(mmDIDT_IND_INDEX, (reg));
 161	WREG32(mmDIDT_IND_DATA, (v));
 162	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 163}
 164
 165static const u32 bonaire_golden_spm_registers[] =
 166{
 167	0xc200, 0xe0ffffff, 0xe0000000
 168};
 169
 170static const u32 bonaire_golden_common_registers[] =
 171{
 172	0x31dc, 0xffffffff, 0x00000800,
 173	0x31dd, 0xffffffff, 0x00000800,
 174	0x31e6, 0xffffffff, 0x00007fbf,
 175	0x31e7, 0xffffffff, 0x00007faf
 176};
 177
 178static const u32 bonaire_golden_registers[] =
 179{
 180	0xcd5, 0x00000333, 0x00000333,
 181	0xcd4, 0x000c0fc0, 0x00040200,
 182	0x2684, 0x00010000, 0x00058208,
 183	0xf000, 0xffff1fff, 0x00140000,
 184	0xf080, 0xfdfc0fff, 0x00000100,
 185	0xf08d, 0x40000000, 0x40000200,
 186	0x260c, 0xffffffff, 0x00000000,
 187	0x260d, 0xf00fffff, 0x00000400,
 188	0x260e, 0x0002021c, 0x00020200,
 189	0x31e, 0x00000080, 0x00000000,
 190	0x16ec, 0x000000f0, 0x00000070,
 191	0x16f0, 0xf0311fff, 0x80300000,
 192	0x263e, 0x73773777, 0x12010001,
 193	0xd43, 0x00810000, 0x408af000,
 194	0x1c0c, 0x31000111, 0x00000011,
 195	0xbd2, 0x73773777, 0x12010001,
 196	0x883, 0x00007fb6, 0x0021a1b1,
 197	0x884, 0x00007fb6, 0x002021b1,
 198	0x860, 0x00007fb6, 0x00002191,
 199	0x886, 0x00007fb6, 0x002121b1,
 200	0x887, 0x00007fb6, 0x002021b1,
 201	0x877, 0x00007fb6, 0x00002191,
 202	0x878, 0x00007fb6, 0x00002191,
 203	0xd8a, 0x0000003f, 0x0000000a,
 204	0xd8b, 0x0000003f, 0x0000000a,
 205	0xab9, 0x00073ffe, 0x000022a2,
 206	0x903, 0x000007ff, 0x00000000,
 207	0x2285, 0xf000003f, 0x00000007,
 208	0x22fc, 0x00002001, 0x00000001,
 209	0x22c9, 0xffffffff, 0x00ffffff,
 210	0xc281, 0x0000ff0f, 0x00000000,
 211	0xa293, 0x07ffffff, 0x06000000,
 212	0x136, 0x00000fff, 0x00000100,
 213	0xf9e, 0x00000001, 0x00000002,
 214	0x2440, 0x03000000, 0x0362c688,
 215	0x2300, 0x000000ff, 0x00000001,
 216	0x390, 0x00001fff, 0x00001fff,
 217	0x2418, 0x0000007f, 0x00000020,
 218	0x2542, 0x00010000, 0x00010000,
 219	0x2b05, 0x000003ff, 0x000000f3,
 220	0x2b03, 0xffffffff, 0x00001032
 221};
 222
 223static const u32 bonaire_mgcg_cgcg_init[] =
 224{
 225	0x3108, 0xffffffff, 0xfffffffc,
 226	0xc200, 0xffffffff, 0xe0000000,
 227	0xf0a8, 0xffffffff, 0x00000100,
 228	0xf082, 0xffffffff, 0x00000100,
 229	0xf0b0, 0xffffffff, 0xc0000100,
 230	0xf0b2, 0xffffffff, 0xc0000100,
 231	0xf0b1, 0xffffffff, 0xc0000100,
 232	0x1579, 0xffffffff, 0x00600100,
 233	0xf0a0, 0xffffffff, 0x00000100,
 234	0xf085, 0xffffffff, 0x06000100,
 235	0xf088, 0xffffffff, 0x00000100,
 236	0xf086, 0xffffffff, 0x06000100,
 237	0xf081, 0xffffffff, 0x00000100,
 238	0xf0b8, 0xffffffff, 0x00000100,
 239	0xf089, 0xffffffff, 0x00000100,
 240	0xf080, 0xffffffff, 0x00000100,
 241	0xf08c, 0xffffffff, 0x00000100,
 242	0xf08d, 0xffffffff, 0x00000100,
 243	0xf094, 0xffffffff, 0x00000100,
 244	0xf095, 0xffffffff, 0x00000100,
 245	0xf096, 0xffffffff, 0x00000100,
 246	0xf097, 0xffffffff, 0x00000100,
 247	0xf098, 0xffffffff, 0x00000100,
 248	0xf09f, 0xffffffff, 0x00000100,
 249	0xf09e, 0xffffffff, 0x00000100,
 250	0xf084, 0xffffffff, 0x06000100,
 251	0xf0a4, 0xffffffff, 0x00000100,
 252	0xf09d, 0xffffffff, 0x00000100,
 253	0xf0ad, 0xffffffff, 0x00000100,
 254	0xf0ac, 0xffffffff, 0x00000100,
 255	0xf09c, 0xffffffff, 0x00000100,
 256	0xc200, 0xffffffff, 0xe0000000,
 257	0xf008, 0xffffffff, 0x00010000,
 258	0xf009, 0xffffffff, 0x00030002,
 259	0xf00a, 0xffffffff, 0x00040007,
 260	0xf00b, 0xffffffff, 0x00060005,
 261	0xf00c, 0xffffffff, 0x00090008,
 262	0xf00d, 0xffffffff, 0x00010000,
 263	0xf00e, 0xffffffff, 0x00030002,
 264	0xf00f, 0xffffffff, 0x00040007,
 265	0xf010, 0xffffffff, 0x00060005,
 266	0xf011, 0xffffffff, 0x00090008,
 267	0xf012, 0xffffffff, 0x00010000,
 268	0xf013, 0xffffffff, 0x00030002,
 269	0xf014, 0xffffffff, 0x00040007,
 270	0xf015, 0xffffffff, 0x00060005,
 271	0xf016, 0xffffffff, 0x00090008,
 272	0xf017, 0xffffffff, 0x00010000,
 273	0xf018, 0xffffffff, 0x00030002,
 274	0xf019, 0xffffffff, 0x00040007,
 275	0xf01a, 0xffffffff, 0x00060005,
 276	0xf01b, 0xffffffff, 0x00090008,
 277	0xf01c, 0xffffffff, 0x00010000,
 278	0xf01d, 0xffffffff, 0x00030002,
 279	0xf01e, 0xffffffff, 0x00040007,
 280	0xf01f, 0xffffffff, 0x00060005,
 281	0xf020, 0xffffffff, 0x00090008,
 282	0xf021, 0xffffffff, 0x00010000,
 283	0xf022, 0xffffffff, 0x00030002,
 284	0xf023, 0xffffffff, 0x00040007,
 285	0xf024, 0xffffffff, 0x00060005,
 286	0xf025, 0xffffffff, 0x00090008,
 287	0xf026, 0xffffffff, 0x00010000,
 288	0xf027, 0xffffffff, 0x00030002,
 289	0xf028, 0xffffffff, 0x00040007,
 290	0xf029, 0xffffffff, 0x00060005,
 291	0xf02a, 0xffffffff, 0x00090008,
 292	0xf000, 0xffffffff, 0x96e00200,
 293	0x21c2, 0xffffffff, 0x00900100,
 294	0x3109, 0xffffffff, 0x0020003f,
 295	0xe, 0xffffffff, 0x0140001c,
 296	0xf, 0x000f0000, 0x000f0000,
 297	0x88, 0xffffffff, 0xc060000c,
 298	0x89, 0xc0000fff, 0x00000100,
 299	0x3e4, 0xffffffff, 0x00000100,
 300	0x3e6, 0x00000101, 0x00000000,
 301	0x82a, 0xffffffff, 0x00000104,
 302	0x1579, 0xff000fff, 0x00000100,
 303	0xc33, 0xc0000fff, 0x00000104,
 304	0x3079, 0x00000001, 0x00000001,
 305	0x3403, 0xff000ff0, 0x00000100,
 306	0x3603, 0xff000ff0, 0x00000100
 307};
 308
 309static const u32 spectre_golden_spm_registers[] =
 310{
 311	0xc200, 0xe0ffffff, 0xe0000000
 312};
 313
 314static const u32 spectre_golden_common_registers[] =
 315{
 316	0x31dc, 0xffffffff, 0x00000800,
 317	0x31dd, 0xffffffff, 0x00000800,
 318	0x31e6, 0xffffffff, 0x00007fbf,
 319	0x31e7, 0xffffffff, 0x00007faf
 320};
 321
 322static const u32 spectre_golden_registers[] =
 323{
 324	0xf000, 0xffff1fff, 0x96940200,
 325	0xf003, 0xffff0001, 0xff000000,
 326	0xf080, 0xfffc0fff, 0x00000100,
 327	0x1bb6, 0x00010101, 0x00010000,
 328	0x260d, 0xf00fffff, 0x00000400,
 329	0x260e, 0xfffffffc, 0x00020200,
 330	0x16ec, 0x000000f0, 0x00000070,
 331	0x16f0, 0xf0311fff, 0x80300000,
 332	0x263e, 0x73773777, 0x12010001,
 333	0x26df, 0x00ff0000, 0x00fc0000,
 334	0xbd2, 0x73773777, 0x12010001,
 335	0x2285, 0xf000003f, 0x00000007,
 336	0x22c9, 0xffffffff, 0x00ffffff,
 337	0xa0d4, 0x3f3f3fff, 0x00000082,
 338	0xa0d5, 0x0000003f, 0x00000000,
 339	0xf9e, 0x00000001, 0x00000002,
 340	0x244f, 0xffff03df, 0x00000004,
 341	0x31da, 0x00000008, 0x00000008,
 342	0x2300, 0x000008ff, 0x00000800,
 343	0x2542, 0x00010000, 0x00010000,
 344	0x2b03, 0xffffffff, 0x54763210,
 345	0x853e, 0x01ff01ff, 0x00000002,
 346	0x8526, 0x007ff800, 0x00200000,
 347	0x8057, 0xffffffff, 0x00000f40,
 348	0xc24d, 0xffffffff, 0x00000001
 349};
 350
 351static const u32 spectre_mgcg_cgcg_init[] =
 352{
 353	0x3108, 0xffffffff, 0xfffffffc,
 354	0xc200, 0xffffffff, 0xe0000000,
 355	0xf0a8, 0xffffffff, 0x00000100,
 356	0xf082, 0xffffffff, 0x00000100,
 357	0xf0b0, 0xffffffff, 0x00000100,
 358	0xf0b2, 0xffffffff, 0x00000100,
 359	0xf0b1, 0xffffffff, 0x00000100,
 360	0x1579, 0xffffffff, 0x00600100,
 361	0xf0a0, 0xffffffff, 0x00000100,
 362	0xf085, 0xffffffff, 0x06000100,
 363	0xf088, 0xffffffff, 0x00000100,
 364	0xf086, 0xffffffff, 0x06000100,
 365	0xf081, 0xffffffff, 0x00000100,
 366	0xf0b8, 0xffffffff, 0x00000100,
 367	0xf089, 0xffffffff, 0x00000100,
 368	0xf080, 0xffffffff, 0x00000100,
 369	0xf08c, 0xffffffff, 0x00000100,
 370	0xf08d, 0xffffffff, 0x00000100,
 371	0xf094, 0xffffffff, 0x00000100,
 372	0xf095, 0xffffffff, 0x00000100,
 373	0xf096, 0xffffffff, 0x00000100,
 374	0xf097, 0xffffffff, 0x00000100,
 375	0xf098, 0xffffffff, 0x00000100,
 376	0xf09f, 0xffffffff, 0x00000100,
 377	0xf09e, 0xffffffff, 0x00000100,
 378	0xf084, 0xffffffff, 0x06000100,
 379	0xf0a4, 0xffffffff, 0x00000100,
 380	0xf09d, 0xffffffff, 0x00000100,
 381	0xf0ad, 0xffffffff, 0x00000100,
 382	0xf0ac, 0xffffffff, 0x00000100,
 383	0xf09c, 0xffffffff, 0x00000100,
 384	0xc200, 0xffffffff, 0xe0000000,
 385	0xf008, 0xffffffff, 0x00010000,
 386	0xf009, 0xffffffff, 0x00030002,
 387	0xf00a, 0xffffffff, 0x00040007,
 388	0xf00b, 0xffffffff, 0x00060005,
 389	0xf00c, 0xffffffff, 0x00090008,
 390	0xf00d, 0xffffffff, 0x00010000,
 391	0xf00e, 0xffffffff, 0x00030002,
 392	0xf00f, 0xffffffff, 0x00040007,
 393	0xf010, 0xffffffff, 0x00060005,
 394	0xf011, 0xffffffff, 0x00090008,
 395	0xf012, 0xffffffff, 0x00010000,
 396	0xf013, 0xffffffff, 0x00030002,
 397	0xf014, 0xffffffff, 0x00040007,
 398	0xf015, 0xffffffff, 0x00060005,
 399	0xf016, 0xffffffff, 0x00090008,
 400	0xf017, 0xffffffff, 0x00010000,
 401	0xf018, 0xffffffff, 0x00030002,
 402	0xf019, 0xffffffff, 0x00040007,
 403	0xf01a, 0xffffffff, 0x00060005,
 404	0xf01b, 0xffffffff, 0x00090008,
 405	0xf01c, 0xffffffff, 0x00010000,
 406	0xf01d, 0xffffffff, 0x00030002,
 407	0xf01e, 0xffffffff, 0x00040007,
 408	0xf01f, 0xffffffff, 0x00060005,
 409	0xf020, 0xffffffff, 0x00090008,
 410	0xf021, 0xffffffff, 0x00010000,
 411	0xf022, 0xffffffff, 0x00030002,
 412	0xf023, 0xffffffff, 0x00040007,
 413	0xf024, 0xffffffff, 0x00060005,
 414	0xf025, 0xffffffff, 0x00090008,
 415	0xf026, 0xffffffff, 0x00010000,
 416	0xf027, 0xffffffff, 0x00030002,
 417	0xf028, 0xffffffff, 0x00040007,
 418	0xf029, 0xffffffff, 0x00060005,
 419	0xf02a, 0xffffffff, 0x00090008,
 420	0xf02b, 0xffffffff, 0x00010000,
 421	0xf02c, 0xffffffff, 0x00030002,
 422	0xf02d, 0xffffffff, 0x00040007,
 423	0xf02e, 0xffffffff, 0x00060005,
 424	0xf02f, 0xffffffff, 0x00090008,
 425	0xf000, 0xffffffff, 0x96e00200,
 426	0x21c2, 0xffffffff, 0x00900100,
 427	0x3109, 0xffffffff, 0x0020003f,
 428	0xe, 0xffffffff, 0x0140001c,
 429	0xf, 0x000f0000, 0x000f0000,
 430	0x88, 0xffffffff, 0xc060000c,
 431	0x89, 0xc0000fff, 0x00000100,
 432	0x3e4, 0xffffffff, 0x00000100,
 433	0x3e6, 0x00000101, 0x00000000,
 434	0x82a, 0xffffffff, 0x00000104,
 435	0x1579, 0xff000fff, 0x00000100,
 436	0xc33, 0xc0000fff, 0x00000104,
 437	0x3079, 0x00000001, 0x00000001,
 438	0x3403, 0xff000ff0, 0x00000100,
 439	0x3603, 0xff000ff0, 0x00000100
 440};
 441
 442static const u32 kalindi_golden_spm_registers[] =
 443{
 444	0xc200, 0xe0ffffff, 0xe0000000
 445};
 446
 447static const u32 kalindi_golden_common_registers[] =
 448{
 449	0x31dc, 0xffffffff, 0x00000800,
 450	0x31dd, 0xffffffff, 0x00000800,
 451	0x31e6, 0xffffffff, 0x00007fbf,
 452	0x31e7, 0xffffffff, 0x00007faf
 453};
 454
 455static const u32 kalindi_golden_registers[] =
 456{
 457	0xf000, 0xffffdfff, 0x6e944040,
 458	0x1579, 0xff607fff, 0xfc000100,
 459	0xf088, 0xff000fff, 0x00000100,
 460	0xf089, 0xff000fff, 0x00000100,
 461	0xf080, 0xfffc0fff, 0x00000100,
 462	0x1bb6, 0x00010101, 0x00010000,
 463	0x260c, 0xffffffff, 0x00000000,
 464	0x260d, 0xf00fffff, 0x00000400,
 465	0x16ec, 0x000000f0, 0x00000070,
 466	0x16f0, 0xf0311fff, 0x80300000,
 467	0x263e, 0x73773777, 0x12010001,
 468	0x263f, 0xffffffff, 0x00000010,
 469	0x26df, 0x00ff0000, 0x00fc0000,
 470	0x200c, 0x00001f0f, 0x0000100a,
 471	0xbd2, 0x73773777, 0x12010001,
 472	0x902, 0x000fffff, 0x000c007f,
 473	0x2285, 0xf000003f, 0x00000007,
 474	0x22c9, 0x3fff3fff, 0x00ffcfff,
 475	0xc281, 0x0000ff0f, 0x00000000,
 476	0xa293, 0x07ffffff, 0x06000000,
 477	0x136, 0x00000fff, 0x00000100,
 478	0xf9e, 0x00000001, 0x00000002,
 479	0x31da, 0x00000008, 0x00000008,
 480	0x2300, 0x000000ff, 0x00000003,
 481	0x853e, 0x01ff01ff, 0x00000002,
 482	0x8526, 0x007ff800, 0x00200000,
 483	0x8057, 0xffffffff, 0x00000f40,
 484	0x2231, 0x001f3ae3, 0x00000082,
 485	0x2235, 0x0000001f, 0x00000010,
 486	0xc24d, 0xffffffff, 0x00000000
 487};
 488
 489static const u32 kalindi_mgcg_cgcg_init[] =
 490{
 491	0x3108, 0xffffffff, 0xfffffffc,
 492	0xc200, 0xffffffff, 0xe0000000,
 493	0xf0a8, 0xffffffff, 0x00000100,
 494	0xf082, 0xffffffff, 0x00000100,
 495	0xf0b0, 0xffffffff, 0x00000100,
 496	0xf0b2, 0xffffffff, 0x00000100,
 497	0xf0b1, 0xffffffff, 0x00000100,
 498	0x1579, 0xffffffff, 0x00600100,
 499	0xf0a0, 0xffffffff, 0x00000100,
 500	0xf085, 0xffffffff, 0x06000100,
 501	0xf088, 0xffffffff, 0x00000100,
 502	0xf086, 0xffffffff, 0x06000100,
 503	0xf081, 0xffffffff, 0x00000100,
 504	0xf0b8, 0xffffffff, 0x00000100,
 505	0xf089, 0xffffffff, 0x00000100,
 506	0xf080, 0xffffffff, 0x00000100,
 507	0xf08c, 0xffffffff, 0x00000100,
 508	0xf08d, 0xffffffff, 0x00000100,
 509	0xf094, 0xffffffff, 0x00000100,
 510	0xf095, 0xffffffff, 0x00000100,
 511	0xf096, 0xffffffff, 0x00000100,
 512	0xf097, 0xffffffff, 0x00000100,
 513	0xf098, 0xffffffff, 0x00000100,
 514	0xf09f, 0xffffffff, 0x00000100,
 515	0xf09e, 0xffffffff, 0x00000100,
 516	0xf084, 0xffffffff, 0x06000100,
 517	0xf0a4, 0xffffffff, 0x00000100,
 518	0xf09d, 0xffffffff, 0x00000100,
 519	0xf0ad, 0xffffffff, 0x00000100,
 520	0xf0ac, 0xffffffff, 0x00000100,
 521	0xf09c, 0xffffffff, 0x00000100,
 522	0xc200, 0xffffffff, 0xe0000000,
 523	0xf008, 0xffffffff, 0x00010000,
 524	0xf009, 0xffffffff, 0x00030002,
 525	0xf00a, 0xffffffff, 0x00040007,
 526	0xf00b, 0xffffffff, 0x00060005,
 527	0xf00c, 0xffffffff, 0x00090008,
 528	0xf00d, 0xffffffff, 0x00010000,
 529	0xf00e, 0xffffffff, 0x00030002,
 530	0xf00f, 0xffffffff, 0x00040007,
 531	0xf010, 0xffffffff, 0x00060005,
 532	0xf011, 0xffffffff, 0x00090008,
 533	0xf000, 0xffffffff, 0x96e00200,
 534	0x21c2, 0xffffffff, 0x00900100,
 535	0x3109, 0xffffffff, 0x0020003f,
 536	0xe, 0xffffffff, 0x0140001c,
 537	0xf, 0x000f0000, 0x000f0000,
 538	0x88, 0xffffffff, 0xc060000c,
 539	0x89, 0xc0000fff, 0x00000100,
 540	0x82a, 0xffffffff, 0x00000104,
 541	0x1579, 0xff000fff, 0x00000100,
 542	0xc33, 0xc0000fff, 0x00000104,
 543	0x3079, 0x00000001, 0x00000001,
 544	0x3403, 0xff000ff0, 0x00000100,
 545	0x3603, 0xff000ff0, 0x00000100
 546};
 547
 548static const u32 hawaii_golden_spm_registers[] =
 549{
 550	0xc200, 0xe0ffffff, 0xe0000000
 551};
 552
 553static const u32 hawaii_golden_common_registers[] =
 554{
 555	0xc200, 0xffffffff, 0xe0000000,
 556	0xa0d4, 0xffffffff, 0x3a00161a,
 557	0xa0d5, 0xffffffff, 0x0000002e,
 558	0x2684, 0xffffffff, 0x00018208,
 559	0x263e, 0xffffffff, 0x12011003
 560};
 561
 562static const u32 hawaii_golden_registers[] =
 563{
 564	0xcd5, 0x00000333, 0x00000333,
 565	0x2684, 0x00010000, 0x00058208,
 566	0x260c, 0xffffffff, 0x00000000,
 567	0x260d, 0xf00fffff, 0x00000400,
 568	0x260e, 0x0002021c, 0x00020200,
 569	0x31e, 0x00000080, 0x00000000,
 570	0x16ec, 0x000000f0, 0x00000070,
 571	0x16f0, 0xf0311fff, 0x80300000,
 572	0xd43, 0x00810000, 0x408af000,
 573	0x1c0c, 0x31000111, 0x00000011,
 574	0xbd2, 0x73773777, 0x12010001,
 575	0x848, 0x0000007f, 0x0000001b,
 576	0x877, 0x00007fb6, 0x00002191,
 577	0xd8a, 0x0000003f, 0x0000000a,
 578	0xd8b, 0x0000003f, 0x0000000a,
 579	0xab9, 0x00073ffe, 0x000022a2,
 580	0x903, 0x000007ff, 0x00000000,
 581	0x22fc, 0x00002001, 0x00000001,
 582	0x22c9, 0xffffffff, 0x00ffffff,
 583	0xc281, 0x0000ff0f, 0x00000000,
 584	0xa293, 0x07ffffff, 0x06000000,
 585	0xf9e, 0x00000001, 0x00000002,
 586	0x31da, 0x00000008, 0x00000008,
 587	0x31dc, 0x00000f00, 0x00000800,
 588	0x31dd, 0x00000f00, 0x00000800,
 589	0x31e6, 0x00ffffff, 0x00ff7fbf,
 590	0x31e7, 0x00ffffff, 0x00ff7faf,
 591	0x2300, 0x000000ff, 0x00000800,
 592	0x390, 0x00001fff, 0x00001fff,
 593	0x2418, 0x0000007f, 0x00000020,
 594	0x2542, 0x00010000, 0x00010000,
 595	0x2b80, 0x00100000, 0x000ff07c,
 596	0x2b05, 0x000003ff, 0x0000000f,
 597	0x2b04, 0xffffffff, 0x7564fdec,
 598	0x2b03, 0xffffffff, 0x3120b9a8,
 599	0x2b02, 0x20000000, 0x0f9c0000
 600};
 601
 602static const u32 hawaii_mgcg_cgcg_init[] =
 603{
 604	0x3108, 0xffffffff, 0xfffffffd,
 605	0xc200, 0xffffffff, 0xe0000000,
 606	0xf0a8, 0xffffffff, 0x00000100,
 607	0xf082, 0xffffffff, 0x00000100,
 608	0xf0b0, 0xffffffff, 0x00000100,
 609	0xf0b2, 0xffffffff, 0x00000100,
 610	0xf0b1, 0xffffffff, 0x00000100,
 611	0x1579, 0xffffffff, 0x00200100,
 612	0xf0a0, 0xffffffff, 0x00000100,
 613	0xf085, 0xffffffff, 0x06000100,
 614	0xf088, 0xffffffff, 0x00000100,
 615	0xf086, 0xffffffff, 0x06000100,
 616	0xf081, 0xffffffff, 0x00000100,
 617	0xf0b8, 0xffffffff, 0x00000100,
 618	0xf089, 0xffffffff, 0x00000100,
 619	0xf080, 0xffffffff, 0x00000100,
 620	0xf08c, 0xffffffff, 0x00000100,
 621	0xf08d, 0xffffffff, 0x00000100,
 622	0xf094, 0xffffffff, 0x00000100,
 623	0xf095, 0xffffffff, 0x00000100,
 624	0xf096, 0xffffffff, 0x00000100,
 625	0xf097, 0xffffffff, 0x00000100,
 626	0xf098, 0xffffffff, 0x00000100,
 627	0xf09f, 0xffffffff, 0x00000100,
 628	0xf09e, 0xffffffff, 0x00000100,
 629	0xf084, 0xffffffff, 0x06000100,
 630	0xf0a4, 0xffffffff, 0x00000100,
 631	0xf09d, 0xffffffff, 0x00000100,
 632	0xf0ad, 0xffffffff, 0x00000100,
 633	0xf0ac, 0xffffffff, 0x00000100,
 634	0xf09c, 0xffffffff, 0x00000100,
 635	0xc200, 0xffffffff, 0xe0000000,
 636	0xf008, 0xffffffff, 0x00010000,
 637	0xf009, 0xffffffff, 0x00030002,
 638	0xf00a, 0xffffffff, 0x00040007,
 639	0xf00b, 0xffffffff, 0x00060005,
 640	0xf00c, 0xffffffff, 0x00090008,
 641	0xf00d, 0xffffffff, 0x00010000,
 642	0xf00e, 0xffffffff, 0x00030002,
 643	0xf00f, 0xffffffff, 0x00040007,
 644	0xf010, 0xffffffff, 0x00060005,
 645	0xf011, 0xffffffff, 0x00090008,
 646	0xf012, 0xffffffff, 0x00010000,
 647	0xf013, 0xffffffff, 0x00030002,
 648	0xf014, 0xffffffff, 0x00040007,
 649	0xf015, 0xffffffff, 0x00060005,
 650	0xf016, 0xffffffff, 0x00090008,
 651	0xf017, 0xffffffff, 0x00010000,
 652	0xf018, 0xffffffff, 0x00030002,
 653	0xf019, 0xffffffff, 0x00040007,
 654	0xf01a, 0xffffffff, 0x00060005,
 655	0xf01b, 0xffffffff, 0x00090008,
 656	0xf01c, 0xffffffff, 0x00010000,
 657	0xf01d, 0xffffffff, 0x00030002,
 658	0xf01e, 0xffffffff, 0x00040007,
 659	0xf01f, 0xffffffff, 0x00060005,
 660	0xf020, 0xffffffff, 0x00090008,
 661	0xf021, 0xffffffff, 0x00010000,
 662	0xf022, 0xffffffff, 0x00030002,
 663	0xf023, 0xffffffff, 0x00040007,
 664	0xf024, 0xffffffff, 0x00060005,
 665	0xf025, 0xffffffff, 0x00090008,
 666	0xf026, 0xffffffff, 0x00010000,
 667	0xf027, 0xffffffff, 0x00030002,
 668	0xf028, 0xffffffff, 0x00040007,
 669	0xf029, 0xffffffff, 0x00060005,
 670	0xf02a, 0xffffffff, 0x00090008,
 671	0xf02b, 0xffffffff, 0x00010000,
 672	0xf02c, 0xffffffff, 0x00030002,
 673	0xf02d, 0xffffffff, 0x00040007,
 674	0xf02e, 0xffffffff, 0x00060005,
 675	0xf02f, 0xffffffff, 0x00090008,
 676	0xf030, 0xffffffff, 0x00010000,
 677	0xf031, 0xffffffff, 0x00030002,
 678	0xf032, 0xffffffff, 0x00040007,
 679	0xf033, 0xffffffff, 0x00060005,
 680	0xf034, 0xffffffff, 0x00090008,
 681	0xf035, 0xffffffff, 0x00010000,
 682	0xf036, 0xffffffff, 0x00030002,
 683	0xf037, 0xffffffff, 0x00040007,
 684	0xf038, 0xffffffff, 0x00060005,
 685	0xf039, 0xffffffff, 0x00090008,
 686	0xf03a, 0xffffffff, 0x00010000,
 687	0xf03b, 0xffffffff, 0x00030002,
 688	0xf03c, 0xffffffff, 0x00040007,
 689	0xf03d, 0xffffffff, 0x00060005,
 690	0xf03e, 0xffffffff, 0x00090008,
 691	0x30c6, 0xffffffff, 0x00020200,
 692	0xcd4, 0xffffffff, 0x00000200,
 693	0x570, 0xffffffff, 0x00000400,
 694	0x157a, 0xffffffff, 0x00000000,
 695	0xbd4, 0xffffffff, 0x00000902,
 696	0xf000, 0xffffffff, 0x96940200,
 697	0x21c2, 0xffffffff, 0x00900100,
 698	0x3109, 0xffffffff, 0x0020003f,
 699	0xe, 0xffffffff, 0x0140001c,
 700	0xf, 0x000f0000, 0x000f0000,
 701	0x88, 0xffffffff, 0xc060000c,
 702	0x89, 0xc0000fff, 0x00000100,
 703	0x3e4, 0xffffffff, 0x00000100,
 704	0x3e6, 0x00000101, 0x00000000,
 705	0x82a, 0xffffffff, 0x00000104,
 706	0x1579, 0xff000fff, 0x00000100,
 707	0xc33, 0xc0000fff, 0x00000104,
 708	0x3079, 0x00000001, 0x00000001,
 709	0x3403, 0xff000ff0, 0x00000100,
 710	0x3603, 0xff000ff0, 0x00000100
 711};
 712
 713static const u32 godavari_golden_registers[] =
 714{
 715	0x1579, 0xff607fff, 0xfc000100,
 716	0x1bb6, 0x00010101, 0x00010000,
 717	0x260c, 0xffffffff, 0x00000000,
 718	0x260c0, 0xf00fffff, 0x00000400,
 719	0x184c, 0xffffffff, 0x00010000,
 720	0x16ec, 0x000000f0, 0x00000070,
 721	0x16f0, 0xf0311fff, 0x80300000,
 722	0x263e, 0x73773777, 0x12010001,
 723	0x263f, 0xffffffff, 0x00000010,
 724	0x200c, 0x00001f0f, 0x0000100a,
 725	0xbd2, 0x73773777, 0x12010001,
 726	0x902, 0x000fffff, 0x000c007f,
 727	0x2285, 0xf000003f, 0x00000007,
 728	0x22c9, 0xffffffff, 0x00ff0fff,
 729	0xc281, 0x0000ff0f, 0x00000000,
 730	0xa293, 0x07ffffff, 0x06000000,
 731	0x136, 0x00000fff, 0x00000100,
 732	0x3405, 0x00010000, 0x00810001,
 733	0x3605, 0x00010000, 0x00810001,
 734	0xf9e, 0x00000001, 0x00000002,
 735	0x31da, 0x00000008, 0x00000008,
 736	0x31dc, 0x00000f00, 0x00000800,
 737	0x31dd, 0x00000f00, 0x00000800,
 738	0x31e6, 0x00ffffff, 0x00ff7fbf,
 739	0x31e7, 0x00ffffff, 0x00ff7faf,
 740	0x2300, 0x000000ff, 0x00000001,
 741	0x853e, 0x01ff01ff, 0x00000002,
 742	0x8526, 0x007ff800, 0x00200000,
 743	0x8057, 0xffffffff, 0x00000f40,
 744	0x2231, 0x001f3ae3, 0x00000082,
 745	0x2235, 0x0000001f, 0x00000010,
 746	0xc24d, 0xffffffff, 0x00000000
 747};
 748
 749static void cik_init_golden_registers(struct amdgpu_device *adev)
 750{
 751	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 752	mutex_lock(&adev->grbm_idx_mutex);
 753
 754	switch (adev->asic_type) {
 755	case CHIP_BONAIRE:
 756		amdgpu_program_register_sequence(adev,
 757						 bonaire_mgcg_cgcg_init,
 758						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 759		amdgpu_program_register_sequence(adev,
 760						 bonaire_golden_registers,
 761						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
 762		amdgpu_program_register_sequence(adev,
 763						 bonaire_golden_common_registers,
 764						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 765		amdgpu_program_register_sequence(adev,
 766						 bonaire_golden_spm_registers,
 767						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 768		break;
 769	case CHIP_KABINI:
 770		amdgpu_program_register_sequence(adev,
 771						 kalindi_mgcg_cgcg_init,
 772						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 773		amdgpu_program_register_sequence(adev,
 774						 kalindi_golden_registers,
 775						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
 776		amdgpu_program_register_sequence(adev,
 777						 kalindi_golden_common_registers,
 778						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 779		amdgpu_program_register_sequence(adev,
 780						 kalindi_golden_spm_registers,
 781						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 782		break;
 783	case CHIP_MULLINS:
 784		amdgpu_program_register_sequence(adev,
 785						 kalindi_mgcg_cgcg_init,
 786						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 787		amdgpu_program_register_sequence(adev,
 788						 godavari_golden_registers,
 789						 (const u32)ARRAY_SIZE(godavari_golden_registers));
 790		amdgpu_program_register_sequence(adev,
 791						 kalindi_golden_common_registers,
 792						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 793		amdgpu_program_register_sequence(adev,
 794						 kalindi_golden_spm_registers,
 795						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 796		break;
 797	case CHIP_KAVERI:
 798		amdgpu_program_register_sequence(adev,
 799						 spectre_mgcg_cgcg_init,
 800						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 801		amdgpu_program_register_sequence(adev,
 802						 spectre_golden_registers,
 803						 (const u32)ARRAY_SIZE(spectre_golden_registers));
 804		amdgpu_program_register_sequence(adev,
 805						 spectre_golden_common_registers,
 806						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 807		amdgpu_program_register_sequence(adev,
 808						 spectre_golden_spm_registers,
 809						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 810		break;
 811	case CHIP_HAWAII:
 812		amdgpu_program_register_sequence(adev,
 813						 hawaii_mgcg_cgcg_init,
 814						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 815		amdgpu_program_register_sequence(adev,
 816						 hawaii_golden_registers,
 817						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
 818		amdgpu_program_register_sequence(adev,
 819						 hawaii_golden_common_registers,
 820						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 821		amdgpu_program_register_sequence(adev,
 822						 hawaii_golden_spm_registers,
 823						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 824		break;
 825	default:
 826		break;
 827	}
 828	mutex_unlock(&adev->grbm_idx_mutex);
 829}
 830
 831/**
 832 * cik_get_xclk - get the xclk
 833 *
 834 * @adev: amdgpu_device pointer
 835 *
 836 * Returns the reference clock used by the gfx engine
 837 * (CIK).
 838 */
 839static u32 cik_get_xclk(struct amdgpu_device *adev)
 840{
 841	u32 reference_clock = adev->clock.spll.reference_freq;
 842
 843	if (adev->flags & AMD_IS_APU) {
 844		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 845			return reference_clock / 2;
 846	} else {
 847		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 848			return reference_clock / 4;
 849	}
 850	return reference_clock;
 851}
 852
 853/**
 854 * cik_srbm_select - select specific register instances
 855 *
 856 * @adev: amdgpu_device pointer
 857 * @me: selected ME (micro engine)
 858 * @pipe: pipe
 859 * @queue: queue
 860 * @vmid: VMID
 861 *
 862 * Switches the currently active registers instances.  Some
 863 * registers are instanced per VMID, others are instanced per
 864 * me/pipe/queue combination.
 865 */
 866void cik_srbm_select(struct amdgpu_device *adev,
 867		     u32 me, u32 pipe, u32 queue, u32 vmid)
 868{
 869	u32 srbm_gfx_cntl =
 870		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 871		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 872		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 873		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 874	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 875}
 876
 877static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 878{
 879	uint32_t tmp;
 880
 881	tmp = RREG32(mmCONFIG_CNTL);
 882	if (state == false)
 883		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 884	else
 885		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 886	WREG32(mmCONFIG_CNTL, tmp);
 887}
 888
 889static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 890{
 891	u32 bus_cntl;
 892	u32 d1vga_control = 0;
 893	u32 d2vga_control = 0;
 894	u32 vga_render_control = 0;
 895	u32 rom_cntl;
 896	bool r;
 897
 898	bus_cntl = RREG32(mmBUS_CNTL);
 899	if (adev->mode_info.num_crtc) {
 900		d1vga_control = RREG32(mmD1VGA_CONTROL);
 901		d2vga_control = RREG32(mmD2VGA_CONTROL);
 902		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 903	}
 904	rom_cntl = RREG32_SMC(ixROM_CNTL);
 905
 906	/* enable the rom */
 907	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 908	if (adev->mode_info.num_crtc) {
 909		/* Disable VGA mode */
 910		WREG32(mmD1VGA_CONTROL,
 911		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 912					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 913		WREG32(mmD2VGA_CONTROL,
 914		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 915					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 916		WREG32(mmVGA_RENDER_CONTROL,
 917		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 918	}
 919	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 920
 921	r = amdgpu_read_bios(adev);
 922
 923	/* restore regs */
 924	WREG32(mmBUS_CNTL, bus_cntl);
 925	if (adev->mode_info.num_crtc) {
 926		WREG32(mmD1VGA_CONTROL, d1vga_control);
 927		WREG32(mmD2VGA_CONTROL, d2vga_control);
 928		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 929	}
 930	WREG32_SMC(ixROM_CNTL, rom_cntl);
 931	return r;
 932}
 933
 934static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 935				   u8 *bios, u32 length_bytes)
 936{
 937	u32 *dw_ptr;
 938	unsigned long flags;
 939	u32 i, length_dw;
 940
 941	if (bios == NULL)
 942		return false;
 943	if (length_bytes == 0)
 944		return false;
 945	/* APU vbios image is part of sbios image */
 946	if (adev->flags & AMD_IS_APU)
 947		return false;
 948
 949	dw_ptr = (u32 *)bios;
 950	length_dw = ALIGN(length_bytes, 4) / 4;
 951	/* take the smc lock since we are using the smc index */
 952	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 953	/* set rom index to 0 */
 954	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 955	WREG32(mmSMC_IND_DATA_0, 0);
 956	/* set index to data for continous read */
 957	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 958	for (i = 0; i < length_dw; i++)
 959		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 960	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961
 962	return true;
 963}
 964
 965static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 966	{mmGRBM_STATUS, false},
 967	{mmGB_ADDR_CONFIG, false},
 968	{mmMC_ARB_RAMCFG, false},
 969	{mmGB_TILE_MODE0, false},
 970	{mmGB_TILE_MODE1, false},
 971	{mmGB_TILE_MODE2, false},
 972	{mmGB_TILE_MODE3, false},
 973	{mmGB_TILE_MODE4, false},
 974	{mmGB_TILE_MODE5, false},
 975	{mmGB_TILE_MODE6, false},
 976	{mmGB_TILE_MODE7, false},
 977	{mmGB_TILE_MODE8, false},
 978	{mmGB_TILE_MODE9, false},
 979	{mmGB_TILE_MODE10, false},
 980	{mmGB_TILE_MODE11, false},
 981	{mmGB_TILE_MODE12, false},
 982	{mmGB_TILE_MODE13, false},
 983	{mmGB_TILE_MODE14, false},
 984	{mmGB_TILE_MODE15, false},
 985	{mmGB_TILE_MODE16, false},
 986	{mmGB_TILE_MODE17, false},
 987	{mmGB_TILE_MODE18, false},
 988	{mmGB_TILE_MODE19, false},
 989	{mmGB_TILE_MODE20, false},
 990	{mmGB_TILE_MODE21, false},
 991	{mmGB_TILE_MODE22, false},
 992	{mmGB_TILE_MODE23, false},
 993	{mmGB_TILE_MODE24, false},
 994	{mmGB_TILE_MODE25, false},
 995	{mmGB_TILE_MODE26, false},
 996	{mmGB_TILE_MODE27, false},
 997	{mmGB_TILE_MODE28, false},
 998	{mmGB_TILE_MODE29, false},
 999	{mmGB_TILE_MODE30, false},
1000	{mmGB_TILE_MODE31, false},
1001	{mmGB_MACROTILE_MODE0, false},
1002	{mmGB_MACROTILE_MODE1, false},
1003	{mmGB_MACROTILE_MODE2, false},
1004	{mmGB_MACROTILE_MODE3, false},
1005	{mmGB_MACROTILE_MODE4, false},
1006	{mmGB_MACROTILE_MODE5, false},
1007	{mmGB_MACROTILE_MODE6, false},
1008	{mmGB_MACROTILE_MODE7, false},
1009	{mmGB_MACROTILE_MODE8, false},
1010	{mmGB_MACROTILE_MODE9, false},
1011	{mmGB_MACROTILE_MODE10, false},
1012	{mmGB_MACROTILE_MODE11, false},
1013	{mmGB_MACROTILE_MODE12, false},
1014	{mmGB_MACROTILE_MODE13, false},
1015	{mmGB_MACROTILE_MODE14, false},
1016	{mmGB_MACROTILE_MODE15, false},
1017	{mmCC_RB_BACKEND_DISABLE, false, true},
1018	{mmGC_USER_RB_BACKEND_DISABLE, false, true},
1019	{mmGB_BACKEND_MAP, false, false},
1020	{mmPA_SC_RASTER_CONFIG, false, true},
1021	{mmPA_SC_RASTER_CONFIG_1, false, true},
1022};
1023
1024static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1025					  u32 se_num, u32 sh_num,
1026					  u32 reg_offset)
1027{
1028	uint32_t val;
1029
1030	mutex_lock(&adev->grbm_idx_mutex);
1031	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1032		gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1033
1034	val = RREG32(reg_offset);
1035
1036	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1037		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1038	mutex_unlock(&adev->grbm_idx_mutex);
1039	return val;
1040}
1041
1042static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1043			     u32 sh_num, u32 reg_offset, u32 *value)
1044{
1045	uint32_t i;
1046
1047	*value = 0;
1048	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1049		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1050			continue;
1051
1052		if (!cik_allowed_read_registers[i].untouched)
1053			*value = cik_allowed_read_registers[i].grbm_indexed ?
1054				 cik_read_indexed_register(adev, se_num,
1055							   sh_num, reg_offset) :
1056				 RREG32(reg_offset);
1057		return 0;
1058	}
1059	return -EINVAL;
1060}
1061
1062struct kv_reset_save_regs {
1063	u32 gmcon_reng_execute;
1064	u32 gmcon_misc;
1065	u32 gmcon_misc3;
1066};
1067
1068static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1069				   struct kv_reset_save_regs *save)
1070{
1071	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1072	save->gmcon_misc = RREG32(mmGMCON_MISC);
1073	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1074
1075	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1076		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1077	WREG32(mmGMCON_MISC, save->gmcon_misc &
1078		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1079			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1080}
1081
1082static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1083				      struct kv_reset_save_regs *save)
1084{
1085	int i;
1086
1087	WREG32(mmGMCON_PGFSM_WRITE, 0);
1088	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1089
1090	for (i = 0; i < 5; i++)
1091		WREG32(mmGMCON_PGFSM_WRITE, 0);
1092
1093	WREG32(mmGMCON_PGFSM_WRITE, 0);
1094	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1095
1096	for (i = 0; i < 5; i++)
1097		WREG32(mmGMCON_PGFSM_WRITE, 0);
1098
1099	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1100	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1101
1102	for (i = 0; i < 5; i++)
1103		WREG32(mmGMCON_PGFSM_WRITE, 0);
1104
1105	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1106	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1107
1108	for (i = 0; i < 5; i++)
1109		WREG32(mmGMCON_PGFSM_WRITE, 0);
1110
1111	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1112	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1113
1114	for (i = 0; i < 5; i++)
1115		WREG32(mmGMCON_PGFSM_WRITE, 0);
1116
1117	WREG32(mmGMCON_PGFSM_WRITE, 0);
1118	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1119
1120	for (i = 0; i < 5; i++)
1121		WREG32(mmGMCON_PGFSM_WRITE, 0);
1122
1123	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1124	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1125
1126	for (i = 0; i < 5; i++)
1127		WREG32(mmGMCON_PGFSM_WRITE, 0);
1128
1129	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1130	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1131
1132	for (i = 0; i < 5; i++)
1133		WREG32(mmGMCON_PGFSM_WRITE, 0);
1134
1135	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1136	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1137
1138	for (i = 0; i < 5; i++)
1139		WREG32(mmGMCON_PGFSM_WRITE, 0);
1140
1141	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1142	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1143
1144	for (i = 0; i < 5; i++)
1145		WREG32(mmGMCON_PGFSM_WRITE, 0);
1146
1147	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1148	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1149
1150	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1151	WREG32(mmGMCON_MISC, save->gmcon_misc);
1152	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1153}
1154
1155static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1156{
1157	struct kv_reset_save_regs kv_save = { 0 };
1158	u32 i;
 
1159
1160	dev_info(adev->dev, "GPU pci config reset\n");
1161
1162	if (adev->flags & AMD_IS_APU)
1163		kv_save_regs_for_reset(adev, &kv_save);
1164
1165	/* disable BM */
1166	pci_clear_master(adev->pdev);
1167	/* reset */
1168	amdgpu_pci_config_reset(adev);
1169
1170	udelay(100);
1171
1172	/* wait for asic to come out of reset */
1173	for (i = 0; i < adev->usec_timeout; i++) {
1174		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
 
 
 
1175			break;
 
1176		udelay(1);
1177	}
1178
1179	/* does asic init need to be run first??? */
1180	if (adev->flags & AMD_IS_APU)
1181		kv_restore_regs_for_reset(adev, &kv_save);
1182}
1183
1184static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1185{
1186	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1187
1188	if (hung)
1189		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1190	else
1191		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1192
1193	WREG32(mmBIOS_SCRATCH_3, tmp);
1194}
1195
1196/**
1197 * cik_asic_reset - soft reset GPU
1198 *
1199 * @adev: amdgpu_device pointer
1200 *
1201 * Look up which blocks are hung and attempt
1202 * to reset them.
1203 * Returns 0 for success.
1204 */
1205static int cik_asic_reset(struct amdgpu_device *adev)
1206{
1207	cik_set_bios_scratch_engine_hung(adev, true);
1208
1209	cik_gpu_pci_config_reset(adev);
1210
1211	cik_set_bios_scratch_engine_hung(adev, false);
1212
1213	return 0;
 
 
1214}
1215
1216static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1217			      u32 cntl_reg, u32 status_reg)
1218{
1219	int r, i;
1220	struct atom_clock_dividers dividers;
1221	uint32_t tmp;
1222
1223	r = amdgpu_atombios_get_clock_dividers(adev,
1224					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1225					       clock, false, &dividers);
1226	if (r)
1227		return r;
1228
1229	tmp = RREG32_SMC(cntl_reg);
1230	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1231		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1232	tmp |= dividers.post_divider;
1233	WREG32_SMC(cntl_reg, tmp);
1234
1235	for (i = 0; i < 100; i++) {
1236		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1237			break;
1238		mdelay(10);
1239	}
1240	if (i == 100)
1241		return -ETIMEDOUT;
1242
1243	return 0;
1244}
1245
1246static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1247{
1248	int r = 0;
1249
1250	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1251	if (r)
1252		return r;
1253
1254	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1255	return r;
1256}
1257
1258static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1259{
1260	int r, i;
1261	struct atom_clock_dividers dividers;
1262	u32 tmp;
1263
1264	r = amdgpu_atombios_get_clock_dividers(adev,
1265					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1266					       ecclk, false, &dividers);
1267	if (r)
1268		return r;
1269
1270	for (i = 0; i < 100; i++) {
1271		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1272			break;
1273		mdelay(10);
1274	}
1275	if (i == 100)
1276		return -ETIMEDOUT;
1277
1278	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1279	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1280		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1281	tmp |= dividers.post_divider;
1282	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1283
1284	for (i = 0; i < 100; i++) {
1285		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1286			break;
1287		mdelay(10);
1288	}
1289	if (i == 100)
1290		return -ETIMEDOUT;
1291
1292	return 0;
1293}
1294
1295static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1296{
1297	struct pci_dev *root = adev->pdev->bus->self;
1298	int bridge_pos, gpu_pos;
1299	u32 speed_cntl, current_data_rate;
1300	int i;
1301	u16 tmp16;
1302
1303	if (pci_is_root_bus(adev->pdev->bus))
1304		return;
1305
1306	if (amdgpu_pcie_gen2 == 0)
1307		return;
1308
1309	if (adev->flags & AMD_IS_APU)
1310		return;
1311
1312	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1313					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1314		return;
1315
1316	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1317	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1318		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1319	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1320		if (current_data_rate == 2) {
1321			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1322			return;
1323		}
1324		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1325	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1326		if (current_data_rate == 1) {
1327			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1328			return;
1329		}
1330		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1331	}
1332
1333	bridge_pos = pci_pcie_cap(root);
1334	if (!bridge_pos)
1335		return;
1336
1337	gpu_pos = pci_pcie_cap(adev->pdev);
1338	if (!gpu_pos)
1339		return;
1340
1341	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1342		/* re-try equalization if gen3 is not already enabled */
1343		if (current_data_rate != 2) {
1344			u16 bridge_cfg, gpu_cfg;
1345			u16 bridge_cfg2, gpu_cfg2;
1346			u32 max_lw, current_lw, tmp;
1347
1348			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1349			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1350
1351			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1352			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1353
1354			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1355			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1356
1357			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1358			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1359				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1360			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1361				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1362
1363			if (current_lw < max_lw) {
1364				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1365				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1366					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1367						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1368					tmp |= (max_lw <<
1369						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1370					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1371					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1372					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1373					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1374				}
1375			}
1376
1377			for (i = 0; i < 10; i++) {
1378				/* check status */
1379				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1380				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1381					break;
1382
1383				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1384				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1385
1386				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1387				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1388
1389				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1390				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1391				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1392
1393				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1394				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1395				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1396
1397				mdelay(100);
1398
1399				/* linkctl */
1400				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1401				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1402				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1403				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1404
1405				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1406				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1407				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1408				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1409
1410				/* linkctl2 */
1411				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1412				tmp16 &= ~((1 << 4) | (7 << 9));
1413				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1414				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1415
1416				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1417				tmp16 &= ~((1 << 4) | (7 << 9));
1418				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1419				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1420
1421				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1422				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1423				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1424			}
1425		}
1426	}
1427
1428	/* set the link speed */
1429	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1430		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1431	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1432	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1433
1434	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1435	tmp16 &= ~0xf;
1436	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1437		tmp16 |= 3; /* gen3 */
1438	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1439		tmp16 |= 2; /* gen2 */
1440	else
1441		tmp16 |= 1; /* gen1 */
1442	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1443
1444	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1445	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1446	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1447
1448	for (i = 0; i < adev->usec_timeout; i++) {
1449		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1450		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1451			break;
1452		udelay(1);
1453	}
1454}
1455
1456static void cik_program_aspm(struct amdgpu_device *adev)
1457{
1458	u32 data, orig;
1459	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1460	bool disable_clkreq = false;
1461
1462	if (amdgpu_aspm == 0)
1463		return;
1464
1465	if (pci_is_root_bus(adev->pdev->bus))
1466		return;
1467
1468	/* XXX double check APUs */
1469	if (adev->flags & AMD_IS_APU)
1470		return;
1471
1472	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1473	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1474	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1475		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1476	if (orig != data)
1477		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1478
1479	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1480	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1481	if (orig != data)
1482		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1483
1484	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1485	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1486	if (orig != data)
1487		WREG32_PCIE(ixPCIE_P_CNTL, data);
1488
1489	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1490	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1491		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1492	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1493	if (!disable_l0s)
1494		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1495
1496	if (!disable_l1) {
1497		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1498		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1499		if (orig != data)
1500			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1501
1502		if (!disable_plloff_in_l1) {
1503			bool clk_req_support;
1504
1505			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1506			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1507				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1508			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1509				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1510			if (orig != data)
1511				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1512
1513			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1514			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1515				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1516			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1517				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1518			if (orig != data)
1519				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1520
1521			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1522			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1523				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1524			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1525				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1526			if (orig != data)
1527				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1528
1529			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1530			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1531				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1532			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1533				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1534			if (orig != data)
1535				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1536
1537			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1538			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1539			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1540			if (orig != data)
1541				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1542
1543			if (!disable_clkreq) {
1544				struct pci_dev *root = adev->pdev->bus->self;
1545				u32 lnkcap;
1546
1547				clk_req_support = false;
1548				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1549				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1550					clk_req_support = true;
1551			} else {
1552				clk_req_support = false;
1553			}
1554
1555			if (clk_req_support) {
1556				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1557				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1558					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1559				if (orig != data)
1560					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1561
1562				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1563				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1564					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1565				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1566					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1567				if (orig != data)
1568					WREG32_SMC(ixTHM_CLK_CNTL, data);
1569
1570				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1571				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1572					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1573				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1574					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1575				if (orig != data)
1576					WREG32_SMC(ixMISC_CLK_CTRL, data);
1577
1578				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1579				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1580				if (orig != data)
1581					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1582
1583				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1584				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1585				if (orig != data)
1586					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1587
1588				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1589				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1590				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1591				if (orig != data)
1592					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1593			}
1594		}
1595	} else {
1596		if (orig != data)
1597			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1598	}
1599
1600	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1601	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1602		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1603		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1604	if (orig != data)
1605		WREG32_PCIE(ixPCIE_CNTL2, data);
1606
1607	if (!disable_l0s) {
1608		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1609		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1610				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1611			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1612			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1613			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1614				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1615				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1616				if (orig != data)
1617					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1618			}
1619		}
1620	}
1621}
1622
1623static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1624{
1625	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1626		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1627}
1628
1629static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1630{
1631	/* ORDER MATTERS! */
1632	{
1633		.type = AMD_IP_BLOCK_TYPE_COMMON,
1634		.major = 1,
1635		.minor = 0,
1636		.rev = 0,
1637		.funcs = &cik_common_ip_funcs,
1638	},
1639	{
1640		.type = AMD_IP_BLOCK_TYPE_GMC,
1641		.major = 7,
1642		.minor = 0,
1643		.rev = 0,
1644		.funcs = &gmc_v7_0_ip_funcs,
1645	},
1646	{
1647		.type = AMD_IP_BLOCK_TYPE_IH,
1648		.major = 2,
1649		.minor = 0,
1650		.rev = 0,
1651		.funcs = &cik_ih_ip_funcs,
1652	},
1653	{
1654		.type = AMD_IP_BLOCK_TYPE_SMC,
1655		.major = 7,
1656		.minor = 0,
1657		.rev = 0,
1658		.funcs = &amdgpu_pp_ip_funcs,
1659	},
1660	{
1661		.type = AMD_IP_BLOCK_TYPE_DCE,
1662		.major = 8,
1663		.minor = 2,
1664		.rev = 0,
1665		.funcs = &dce_v8_0_ip_funcs,
1666	},
1667	{
1668		.type = AMD_IP_BLOCK_TYPE_GFX,
1669		.major = 7,
1670		.minor = 2,
1671		.rev = 0,
1672		.funcs = &gfx_v7_0_ip_funcs,
1673	},
1674	{
1675		.type = AMD_IP_BLOCK_TYPE_SDMA,
1676		.major = 2,
1677		.minor = 0,
1678		.rev = 0,
1679		.funcs = &cik_sdma_ip_funcs,
1680	},
1681	{
1682		.type = AMD_IP_BLOCK_TYPE_UVD,
1683		.major = 4,
1684		.minor = 2,
1685		.rev = 0,
1686		.funcs = &uvd_v4_2_ip_funcs,
1687	},
1688	{
1689		.type = AMD_IP_BLOCK_TYPE_VCE,
1690		.major = 2,
1691		.minor = 0,
1692		.rev = 0,
1693		.funcs = &vce_v2_0_ip_funcs,
1694	},
1695};
1696
1697static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1698{
1699	/* ORDER MATTERS! */
1700	{
1701		.type = AMD_IP_BLOCK_TYPE_COMMON,
1702		.major = 1,
1703		.minor = 0,
1704		.rev = 0,
1705		.funcs = &cik_common_ip_funcs,
1706	},
1707	{
1708		.type = AMD_IP_BLOCK_TYPE_GMC,
1709		.major = 7,
1710		.minor = 0,
1711		.rev = 0,
1712		.funcs = &gmc_v7_0_ip_funcs,
1713	},
1714	{
1715		.type = AMD_IP_BLOCK_TYPE_IH,
1716		.major = 2,
1717		.minor = 0,
1718		.rev = 0,
1719		.funcs = &cik_ih_ip_funcs,
1720	},
1721	{
1722		.type = AMD_IP_BLOCK_TYPE_SMC,
1723		.major = 7,
1724		.minor = 0,
1725		.rev = 0,
1726		.funcs = &amdgpu_pp_ip_funcs,
1727	},
1728	{
1729		.type = AMD_IP_BLOCK_TYPE_DCE,
1730		.major = 8,
1731		.minor = 5,
1732		.rev = 0,
1733		.funcs = &dce_v8_0_ip_funcs,
1734	},
1735	{
1736		.type = AMD_IP_BLOCK_TYPE_GFX,
1737		.major = 7,
1738		.minor = 3,
1739		.rev = 0,
1740		.funcs = &gfx_v7_0_ip_funcs,
1741	},
1742	{
1743		.type = AMD_IP_BLOCK_TYPE_SDMA,
1744		.major = 2,
1745		.minor = 0,
1746		.rev = 0,
1747		.funcs = &cik_sdma_ip_funcs,
1748	},
1749	{
1750		.type = AMD_IP_BLOCK_TYPE_UVD,
1751		.major = 4,
1752		.minor = 2,
1753		.rev = 0,
1754		.funcs = &uvd_v4_2_ip_funcs,
1755	},
1756	{
1757		.type = AMD_IP_BLOCK_TYPE_VCE,
1758		.major = 2,
1759		.minor = 0,
1760		.rev = 0,
1761		.funcs = &vce_v2_0_ip_funcs,
1762	},
1763};
1764
1765static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1766{
1767	/* ORDER MATTERS! */
1768	{
1769		.type = AMD_IP_BLOCK_TYPE_COMMON,
1770		.major = 1,
1771		.minor = 0,
1772		.rev = 0,
1773		.funcs = &cik_common_ip_funcs,
1774	},
1775	{
1776		.type = AMD_IP_BLOCK_TYPE_GMC,
1777		.major = 7,
1778		.minor = 0,
1779		.rev = 0,
1780		.funcs = &gmc_v7_0_ip_funcs,
1781	},
1782	{
1783		.type = AMD_IP_BLOCK_TYPE_IH,
1784		.major = 2,
1785		.minor = 0,
1786		.rev = 0,
1787		.funcs = &cik_ih_ip_funcs,
1788	},
1789	{
1790		.type = AMD_IP_BLOCK_TYPE_SMC,
1791		.major = 7,
1792		.minor = 0,
1793		.rev = 0,
1794		.funcs = &amdgpu_pp_ip_funcs,
1795	},
1796	{
1797		.type = AMD_IP_BLOCK_TYPE_DCE,
1798		.major = 8,
1799		.minor = 3,
1800		.rev = 0,
1801		.funcs = &dce_v8_0_ip_funcs,
1802	},
1803	{
1804		.type = AMD_IP_BLOCK_TYPE_GFX,
1805		.major = 7,
1806		.minor = 2,
1807		.rev = 0,
1808		.funcs = &gfx_v7_0_ip_funcs,
1809	},
1810	{
1811		.type = AMD_IP_BLOCK_TYPE_SDMA,
1812		.major = 2,
1813		.minor = 0,
1814		.rev = 0,
1815		.funcs = &cik_sdma_ip_funcs,
1816	},
1817	{
1818		.type = AMD_IP_BLOCK_TYPE_UVD,
1819		.major = 4,
1820		.minor = 2,
1821		.rev = 0,
1822		.funcs = &uvd_v4_2_ip_funcs,
1823	},
1824	{
1825		.type = AMD_IP_BLOCK_TYPE_VCE,
1826		.major = 2,
1827		.minor = 0,
1828		.rev = 0,
1829		.funcs = &vce_v2_0_ip_funcs,
1830	},
1831};
1832
1833static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1834{
1835	/* ORDER MATTERS! */
1836	{
1837		.type = AMD_IP_BLOCK_TYPE_COMMON,
1838		.major = 1,
1839		.minor = 0,
1840		.rev = 0,
1841		.funcs = &cik_common_ip_funcs,
1842	},
1843	{
1844		.type = AMD_IP_BLOCK_TYPE_GMC,
1845		.major = 7,
1846		.minor = 0,
1847		.rev = 0,
1848		.funcs = &gmc_v7_0_ip_funcs,
1849	},
1850	{
1851		.type = AMD_IP_BLOCK_TYPE_IH,
1852		.major = 2,
1853		.minor = 0,
1854		.rev = 0,
1855		.funcs = &cik_ih_ip_funcs,
1856	},
1857	{
1858		.type = AMD_IP_BLOCK_TYPE_SMC,
1859		.major = 7,
1860		.minor = 0,
1861		.rev = 0,
1862		.funcs = &amdgpu_pp_ip_funcs,
1863	},
1864	{
1865		.type = AMD_IP_BLOCK_TYPE_DCE,
1866		.major = 8,
1867		.minor = 3,
1868		.rev = 0,
1869		.funcs = &dce_v8_0_ip_funcs,
1870	},
1871	{
1872		.type = AMD_IP_BLOCK_TYPE_GFX,
1873		.major = 7,
1874		.minor = 2,
1875		.rev = 0,
1876		.funcs = &gfx_v7_0_ip_funcs,
1877	},
1878	{
1879		.type = AMD_IP_BLOCK_TYPE_SDMA,
1880		.major = 2,
1881		.minor = 0,
1882		.rev = 0,
1883		.funcs = &cik_sdma_ip_funcs,
1884	},
1885	{
1886		.type = AMD_IP_BLOCK_TYPE_UVD,
1887		.major = 4,
1888		.minor = 2,
1889		.rev = 0,
1890		.funcs = &uvd_v4_2_ip_funcs,
1891	},
1892	{
1893		.type = AMD_IP_BLOCK_TYPE_VCE,
1894		.major = 2,
1895		.minor = 0,
1896		.rev = 0,
1897		.funcs = &vce_v2_0_ip_funcs,
1898	},
1899};
1900
1901static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1902{
1903	/* ORDER MATTERS! */
1904	{
1905		.type = AMD_IP_BLOCK_TYPE_COMMON,
1906		.major = 1,
1907		.minor = 0,
1908		.rev = 0,
1909		.funcs = &cik_common_ip_funcs,
1910	},
1911	{
1912		.type = AMD_IP_BLOCK_TYPE_GMC,
1913		.major = 7,
1914		.minor = 0,
1915		.rev = 0,
1916		.funcs = &gmc_v7_0_ip_funcs,
1917	},
1918	{
1919		.type = AMD_IP_BLOCK_TYPE_IH,
1920		.major = 2,
1921		.minor = 0,
1922		.rev = 0,
1923		.funcs = &cik_ih_ip_funcs,
1924	},
1925	{
1926		.type = AMD_IP_BLOCK_TYPE_SMC,
1927		.major = 7,
1928		.minor = 0,
1929		.rev = 0,
1930		.funcs = &amdgpu_pp_ip_funcs,
1931	},
1932	{
1933		.type = AMD_IP_BLOCK_TYPE_DCE,
1934		.major = 8,
1935		.minor = 1,
1936		.rev = 0,
1937		.funcs = &dce_v8_0_ip_funcs,
1938	},
1939	{
1940		.type = AMD_IP_BLOCK_TYPE_GFX,
1941		.major = 7,
1942		.minor = 1,
1943		.rev = 0,
1944		.funcs = &gfx_v7_0_ip_funcs,
1945	},
1946	{
1947		.type = AMD_IP_BLOCK_TYPE_SDMA,
1948		.major = 2,
1949		.minor = 0,
1950		.rev = 0,
1951		.funcs = &cik_sdma_ip_funcs,
1952	},
1953	{
1954		.type = AMD_IP_BLOCK_TYPE_UVD,
1955		.major = 4,
1956		.minor = 2,
1957		.rev = 0,
1958		.funcs = &uvd_v4_2_ip_funcs,
1959	},
1960	{
1961		.type = AMD_IP_BLOCK_TYPE_VCE,
1962		.major = 2,
1963		.minor = 0,
1964		.rev = 0,
1965		.funcs = &vce_v2_0_ip_funcs,
1966	},
1967};
1968
1969int cik_set_ip_blocks(struct amdgpu_device *adev)
1970{
1971	switch (adev->asic_type) {
1972	case CHIP_BONAIRE:
1973		adev->ip_blocks = bonaire_ip_blocks;
1974		adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1975		break;
1976	case CHIP_HAWAII:
1977		adev->ip_blocks = hawaii_ip_blocks;
1978		adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1979		break;
1980	case CHIP_KAVERI:
1981		adev->ip_blocks = kaveri_ip_blocks;
1982		adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1983		break;
1984	case CHIP_KABINI:
1985		adev->ip_blocks = kabini_ip_blocks;
1986		adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
1987		break;
1988	case CHIP_MULLINS:
1989		adev->ip_blocks = mullins_ip_blocks;
1990		adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
1991		break;
1992	default:
1993		/* FIXME: not supported yet */
1994		return -EINVAL;
1995	}
1996
1997	return 0;
1998}
1999
2000static const struct amdgpu_asic_funcs cik_asic_funcs =
2001{
2002	.read_disabled_bios = &cik_read_disabled_bios,
2003	.read_bios_from_rom = &cik_read_bios_from_rom,
 
2004	.read_register = &cik_read_register,
2005	.reset = &cik_asic_reset,
2006	.set_vga_state = &cik_vga_set_state,
2007	.get_xclk = &cik_get_xclk,
2008	.set_uvd_clocks = &cik_set_uvd_clocks,
2009	.set_vce_clocks = &cik_set_vce_clocks,
2010	.get_cu_info = &gfx_v7_0_get_cu_info,
2011	/* these should be moved to their own ip modules */
2012	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2013	.wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2014};
2015
2016static int cik_common_early_init(void *handle)
2017{
2018	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2019
2020	adev->smc_rreg = &cik_smc_rreg;
2021	adev->smc_wreg = &cik_smc_wreg;
2022	adev->pcie_rreg = &cik_pcie_rreg;
2023	adev->pcie_wreg = &cik_pcie_wreg;
2024	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2025	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2026	adev->didt_rreg = &cik_didt_rreg;
2027	adev->didt_wreg = &cik_didt_wreg;
2028
2029	adev->asic_funcs = &cik_asic_funcs;
2030
2031	adev->rev_id = cik_get_rev_id(adev);
2032	adev->external_rev_id = 0xFF;
2033	switch (adev->asic_type) {
2034	case CHIP_BONAIRE:
2035		adev->cg_flags =
2036			AMD_CG_SUPPORT_GFX_MGCG |
2037			AMD_CG_SUPPORT_GFX_MGLS |
2038			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2039			AMD_CG_SUPPORT_GFX_CGLS |
2040			AMD_CG_SUPPORT_GFX_CGTS |
2041			AMD_CG_SUPPORT_GFX_CGTS_LS |
2042			AMD_CG_SUPPORT_GFX_CP_LS |
2043			AMD_CG_SUPPORT_MC_LS |
2044			AMD_CG_SUPPORT_MC_MGCG |
2045			AMD_CG_SUPPORT_SDMA_MGCG |
2046			AMD_CG_SUPPORT_SDMA_LS |
2047			AMD_CG_SUPPORT_BIF_LS |
2048			AMD_CG_SUPPORT_VCE_MGCG |
2049			AMD_CG_SUPPORT_UVD_MGCG |
2050			AMD_CG_SUPPORT_HDP_LS |
2051			AMD_CG_SUPPORT_HDP_MGCG;
2052		adev->pg_flags = 0;
2053		adev->external_rev_id = adev->rev_id + 0x14;
2054		break;
2055	case CHIP_HAWAII:
2056		adev->cg_flags =
2057			AMD_CG_SUPPORT_GFX_MGCG |
2058			AMD_CG_SUPPORT_GFX_MGLS |
2059			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2060			AMD_CG_SUPPORT_GFX_CGLS |
2061			AMD_CG_SUPPORT_GFX_CGTS |
2062			AMD_CG_SUPPORT_GFX_CP_LS |
2063			AMD_CG_SUPPORT_MC_LS |
2064			AMD_CG_SUPPORT_MC_MGCG |
2065			AMD_CG_SUPPORT_SDMA_MGCG |
2066			AMD_CG_SUPPORT_SDMA_LS |
2067			AMD_CG_SUPPORT_BIF_LS |
2068			AMD_CG_SUPPORT_VCE_MGCG |
2069			AMD_CG_SUPPORT_UVD_MGCG |
2070			AMD_CG_SUPPORT_HDP_LS |
2071			AMD_CG_SUPPORT_HDP_MGCG;
2072		adev->pg_flags = 0;
2073		adev->external_rev_id = 0x28;
2074		break;
2075	case CHIP_KAVERI:
2076		adev->cg_flags =
2077			AMD_CG_SUPPORT_GFX_MGCG |
2078			AMD_CG_SUPPORT_GFX_MGLS |
2079			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2080			AMD_CG_SUPPORT_GFX_CGLS |
2081			AMD_CG_SUPPORT_GFX_CGTS |
2082			AMD_CG_SUPPORT_GFX_CGTS_LS |
2083			AMD_CG_SUPPORT_GFX_CP_LS |
2084			AMD_CG_SUPPORT_SDMA_MGCG |
2085			AMD_CG_SUPPORT_SDMA_LS |
2086			AMD_CG_SUPPORT_BIF_LS |
2087			AMD_CG_SUPPORT_VCE_MGCG |
2088			AMD_CG_SUPPORT_UVD_MGCG |
2089			AMD_CG_SUPPORT_HDP_LS |
2090			AMD_CG_SUPPORT_HDP_MGCG;
2091		adev->pg_flags =
2092			/*AMD_PG_SUPPORT_GFX_PG |
2093			  AMD_PG_SUPPORT_GFX_SMG |
2094			  AMD_PG_SUPPORT_GFX_DMG |*/
2095			AMD_PG_SUPPORT_UVD |
2096			/*AMD_PG_SUPPORT_VCE |
2097			  AMD_PG_SUPPORT_CP |
2098			  AMD_PG_SUPPORT_GDS |
2099			  AMD_PG_SUPPORT_RLC_SMU_HS |
2100			  AMD_PG_SUPPORT_ACP |
2101			  AMD_PG_SUPPORT_SAMU |*/
2102			0;
2103		if (adev->pdev->device == 0x1312 ||
2104			adev->pdev->device == 0x1316 ||
2105			adev->pdev->device == 0x1317)
2106			adev->external_rev_id = 0x41;
2107		else
2108			adev->external_rev_id = 0x1;
2109		break;
2110	case CHIP_KABINI:
2111	case CHIP_MULLINS:
2112		adev->cg_flags =
2113			AMD_CG_SUPPORT_GFX_MGCG |
2114			AMD_CG_SUPPORT_GFX_MGLS |
2115			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2116			AMD_CG_SUPPORT_GFX_CGLS |
2117			AMD_CG_SUPPORT_GFX_CGTS |
2118			AMD_CG_SUPPORT_GFX_CGTS_LS |
2119			AMD_CG_SUPPORT_GFX_CP_LS |
2120			AMD_CG_SUPPORT_SDMA_MGCG |
2121			AMD_CG_SUPPORT_SDMA_LS |
2122			AMD_CG_SUPPORT_BIF_LS |
2123			AMD_CG_SUPPORT_VCE_MGCG |
2124			AMD_CG_SUPPORT_UVD_MGCG |
2125			AMD_CG_SUPPORT_HDP_LS |
2126			AMD_CG_SUPPORT_HDP_MGCG;
2127		adev->pg_flags =
2128			/*AMD_PG_SUPPORT_GFX_PG |
2129			  AMD_PG_SUPPORT_GFX_SMG | */
2130			AMD_PG_SUPPORT_UVD |
2131			/*AMD_PG_SUPPORT_VCE |
2132			  AMD_PG_SUPPORT_CP |
2133			  AMD_PG_SUPPORT_GDS |
2134			  AMD_PG_SUPPORT_RLC_SMU_HS |
2135			  AMD_PG_SUPPORT_SAMU |*/
2136			0;
2137		if (adev->asic_type == CHIP_KABINI) {
2138			if (adev->rev_id == 0)
2139				adev->external_rev_id = 0x81;
2140			else if (adev->rev_id == 1)
2141				adev->external_rev_id = 0x82;
2142			else if (adev->rev_id == 2)
2143				adev->external_rev_id = 0x85;
2144		} else
2145			adev->external_rev_id = adev->rev_id + 0xa1;
2146		break;
2147	default:
2148		/* FIXME: not supported yet */
2149		return -EINVAL;
2150	}
2151
2152	amdgpu_get_pcie_info(adev);
2153
2154	return 0;
2155}
2156
2157static int cik_common_sw_init(void *handle)
2158{
2159	return 0;
2160}
2161
2162static int cik_common_sw_fini(void *handle)
2163{
2164	return 0;
2165}
2166
2167static int cik_common_hw_init(void *handle)
2168{
2169	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2170
2171	/* move the golden regs per IP block */
2172	cik_init_golden_registers(adev);
2173	/* enable pcie gen2/3 link */
2174	cik_pcie_gen3_enable(adev);
2175	/* enable aspm */
2176	cik_program_aspm(adev);
2177
2178	return 0;
2179}
2180
2181static int cik_common_hw_fini(void *handle)
2182{
2183	return 0;
2184}
2185
2186static int cik_common_suspend(void *handle)
2187{
2188	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2189
2190	amdgpu_amdkfd_suspend(adev);
2191
2192	return cik_common_hw_fini(adev);
2193}
2194
2195static int cik_common_resume(void *handle)
2196{
2197	int r;
2198	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2199
2200	r = cik_common_hw_init(adev);
2201	if (r)
2202		return r;
2203
2204	return amdgpu_amdkfd_resume(adev);
2205}
2206
2207static bool cik_common_is_idle(void *handle)
2208{
2209	return true;
2210}
2211
2212static int cik_common_wait_for_idle(void *handle)
2213{
2214	return 0;
2215}
2216
2217static void cik_common_print_status(void *handle)
2218{
2219
2220}
2221
2222static int cik_common_soft_reset(void *handle)
2223{
2224	/* XXX hard reset?? */
2225	return 0;
2226}
2227
2228static int cik_common_set_clockgating_state(void *handle,
2229					    enum amd_clockgating_state state)
2230{
2231	return 0;
2232}
2233
2234static int cik_common_set_powergating_state(void *handle,
2235					    enum amd_powergating_state state)
2236{
2237	return 0;
2238}
2239
2240const struct amd_ip_funcs cik_common_ip_funcs = {
 
2241	.early_init = cik_common_early_init,
2242	.late_init = NULL,
2243	.sw_init = cik_common_sw_init,
2244	.sw_fini = cik_common_sw_fini,
2245	.hw_init = cik_common_hw_init,
2246	.hw_fini = cik_common_hw_fini,
2247	.suspend = cik_common_suspend,
2248	.resume = cik_common_resume,
2249	.is_idle = cik_common_is_idle,
2250	.wait_for_idle = cik_common_wait_for_idle,
2251	.soft_reset = cik_common_soft_reset,
2252	.print_status = cik_common_print_status,
2253	.set_clockgating_state = cik_common_set_clockgating_state,
2254	.set_powergating_state = cik_common_set_powergating_state,
2255};