1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_atombios.h"
  31#include "amdgpu_ih.h"
  32#include "amdgpu_uvd.h"
  33#include "amdgpu_vce.h"
  34#include "cikd.h"
  35#include "atom.h"
  36#include "amd_pcie.h"
  37
  38#include "cik.h"
  39#include "gmc_v7_0.h"
  40#include "cik_ih.h"
  41#include "dce_v8_0.h"
  42#include "gfx_v7_0.h"
  43#include "cik_sdma.h"
  44#include "uvd_v4_2.h"
  45#include "vce_v2_0.h"
  46#include "cik_dpm.h"
  47
  48#include "uvd/uvd_4_2_d.h"
  49
  50#include "smu/smu_7_0_1_d.h"
  51#include "smu/smu_7_0_1_sh_mask.h"
  52
  53#include "dce/dce_8_0_d.h"
  54#include "dce/dce_8_0_sh_mask.h"
  55
  56#include "bif/bif_4_1_d.h"
  57#include "bif/bif_4_1_sh_mask.h"
  58
  59#include "gca/gfx_7_2_d.h"
  60#include "gca/gfx_7_2_enum.h"
  61#include "gca/gfx_7_2_sh_mask.h"
  62
  63#include "gmc/gmc_7_1_d.h"
  64#include "gmc/gmc_7_1_sh_mask.h"
  65
  66#include "oss/oss_2_0_d.h"
  67#include "oss/oss_2_0_sh_mask.h"
  68
  69#include "amdgpu_dm.h"
  70#include "amdgpu_amdkfd.h"
 
  71#include "dce_virtual.h"
  72
  73/*
  74 * Indirect registers accessor
  75 */
  76static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  77{
  78	unsigned long flags;
  79	u32 r;
  80
  81	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  82	WREG32(mmPCIE_INDEX, reg);
  83	(void)RREG32(mmPCIE_INDEX);
  84	r = RREG32(mmPCIE_DATA);
  85	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  86	return r;
  87}
  88
  89static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  90{
  91	unsigned long flags;
  92
  93	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  94	WREG32(mmPCIE_INDEX, reg);
  95	(void)RREG32(mmPCIE_INDEX);
  96	WREG32(mmPCIE_DATA, v);
  97	(void)RREG32(mmPCIE_DATA);
  98	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  99}
 100
 101static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 102{
 103	unsigned long flags;
 104	u32 r;
 105
 106	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 107	WREG32(mmSMC_IND_INDEX_0, (reg));
 108	r = RREG32(mmSMC_IND_DATA_0);
 109	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 110	return r;
 111}
 112
 113static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 114{
 115	unsigned long flags;
 116
 117	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 118	WREG32(mmSMC_IND_INDEX_0, (reg));
 119	WREG32(mmSMC_IND_DATA_0, (v));
 120	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 121}
 122
 123static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 124{
 125	unsigned long flags;
 126	u32 r;
 127
 128	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 129	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 130	r = RREG32(mmUVD_CTX_DATA);
 131	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 132	return r;
 133}
 134
 135static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 136{
 137	unsigned long flags;
 138
 139	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 140	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 141	WREG32(mmUVD_CTX_DATA, (v));
 142	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 143}
 144
 145static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 146{
 147	unsigned long flags;
 148	u32 r;
 149
 150	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 151	WREG32(mmDIDT_IND_INDEX, (reg));
 152	r = RREG32(mmDIDT_IND_DATA);
 153	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 154	return r;
 155}
 156
 157static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 158{
 159	unsigned long flags;
 160
 161	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 162	WREG32(mmDIDT_IND_INDEX, (reg));
 163	WREG32(mmDIDT_IND_DATA, (v));
 164	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 165}
 166
 167static const u32 bonaire_golden_spm_registers[] =
 168{
 169	0xc200, 0xe0ffffff, 0xe0000000
 170};
 171
 172static const u32 bonaire_golden_common_registers[] =
 173{
 174	0x31dc, 0xffffffff, 0x00000800,
 175	0x31dd, 0xffffffff, 0x00000800,
 176	0x31e6, 0xffffffff, 0x00007fbf,
 177	0x31e7, 0xffffffff, 0x00007faf
 178};
 179
 180static const u32 bonaire_golden_registers[] =
 181{
 182	0xcd5, 0x00000333, 0x00000333,
 183	0xcd4, 0x000c0fc0, 0x00040200,
 184	0x2684, 0x00010000, 0x00058208,
 185	0xf000, 0xffff1fff, 0x00140000,
 186	0xf080, 0xfdfc0fff, 0x00000100,
 187	0xf08d, 0x40000000, 0x40000200,
 188	0x260c, 0xffffffff, 0x00000000,
 189	0x260d, 0xf00fffff, 0x00000400,
 190	0x260e, 0x0002021c, 0x00020200,
 191	0x31e, 0x00000080, 0x00000000,
 192	0x16ec, 0x000000f0, 0x00000070,
 193	0x16f0, 0xf0311fff, 0x80300000,
 194	0x263e, 0x73773777, 0x12010001,
 195	0xd43, 0x00810000, 0x408af000,
 196	0x1c0c, 0x31000111, 0x00000011,
 197	0xbd2, 0x73773777, 0x12010001,
 198	0x883, 0x00007fb6, 0x0021a1b1,
 199	0x884, 0x00007fb6, 0x002021b1,
 200	0x860, 0x00007fb6, 0x00002191,
 201	0x886, 0x00007fb6, 0x002121b1,
 202	0x887, 0x00007fb6, 0x002021b1,
 203	0x877, 0x00007fb6, 0x00002191,
 204	0x878, 0x00007fb6, 0x00002191,
 205	0xd8a, 0x0000003f, 0x0000000a,
 206	0xd8b, 0x0000003f, 0x0000000a,
 207	0xab9, 0x00073ffe, 0x000022a2,
 208	0x903, 0x000007ff, 0x00000000,
 209	0x2285, 0xf000003f, 0x00000007,
 210	0x22fc, 0x00002001, 0x00000001,
 211	0x22c9, 0xffffffff, 0x00ffffff,
 212	0xc281, 0x0000ff0f, 0x00000000,
 213	0xa293, 0x07ffffff, 0x06000000,
 214	0x136, 0x00000fff, 0x00000100,
 215	0xf9e, 0x00000001, 0x00000002,
 216	0x2440, 0x03000000, 0x0362c688,
 217	0x2300, 0x000000ff, 0x00000001,
 218	0x390, 0x00001fff, 0x00001fff,
 219	0x2418, 0x0000007f, 0x00000020,
 220	0x2542, 0x00010000, 0x00010000,
 221	0x2b05, 0x000003ff, 0x000000f3,
 222	0x2b03, 0xffffffff, 0x00001032
 223};
 224
 225static const u32 bonaire_mgcg_cgcg_init[] =
 226{
 227	0x3108, 0xffffffff, 0xfffffffc,
 228	0xc200, 0xffffffff, 0xe0000000,
 229	0xf0a8, 0xffffffff, 0x00000100,
 230	0xf082, 0xffffffff, 0x00000100,
 231	0xf0b0, 0xffffffff, 0xc0000100,
 232	0xf0b2, 0xffffffff, 0xc0000100,
 233	0xf0b1, 0xffffffff, 0xc0000100,
 234	0x1579, 0xffffffff, 0x00600100,
 235	0xf0a0, 0xffffffff, 0x00000100,
 236	0xf085, 0xffffffff, 0x06000100,
 237	0xf088, 0xffffffff, 0x00000100,
 238	0xf086, 0xffffffff, 0x06000100,
 239	0xf081, 0xffffffff, 0x00000100,
 240	0xf0b8, 0xffffffff, 0x00000100,
 241	0xf089, 0xffffffff, 0x00000100,
 242	0xf080, 0xffffffff, 0x00000100,
 243	0xf08c, 0xffffffff, 0x00000100,
 244	0xf08d, 0xffffffff, 0x00000100,
 245	0xf094, 0xffffffff, 0x00000100,
 246	0xf095, 0xffffffff, 0x00000100,
 247	0xf096, 0xffffffff, 0x00000100,
 248	0xf097, 0xffffffff, 0x00000100,
 249	0xf098, 0xffffffff, 0x00000100,
 250	0xf09f, 0xffffffff, 0x00000100,
 251	0xf09e, 0xffffffff, 0x00000100,
 252	0xf084, 0xffffffff, 0x06000100,
 253	0xf0a4, 0xffffffff, 0x00000100,
 254	0xf09d, 0xffffffff, 0x00000100,
 255	0xf0ad, 0xffffffff, 0x00000100,
 256	0xf0ac, 0xffffffff, 0x00000100,
 257	0xf09c, 0xffffffff, 0x00000100,
 258	0xc200, 0xffffffff, 0xe0000000,
 259	0xf008, 0xffffffff, 0x00010000,
 260	0xf009, 0xffffffff, 0x00030002,
 261	0xf00a, 0xffffffff, 0x00040007,
 262	0xf00b, 0xffffffff, 0x00060005,
 263	0xf00c, 0xffffffff, 0x00090008,
 264	0xf00d, 0xffffffff, 0x00010000,
 265	0xf00e, 0xffffffff, 0x00030002,
 266	0xf00f, 0xffffffff, 0x00040007,
 267	0xf010, 0xffffffff, 0x00060005,
 268	0xf011, 0xffffffff, 0x00090008,
 269	0xf012, 0xffffffff, 0x00010000,
 270	0xf013, 0xffffffff, 0x00030002,
 271	0xf014, 0xffffffff, 0x00040007,
 272	0xf015, 0xffffffff, 0x00060005,
 273	0xf016, 0xffffffff, 0x00090008,
 274	0xf017, 0xffffffff, 0x00010000,
 275	0xf018, 0xffffffff, 0x00030002,
 276	0xf019, 0xffffffff, 0x00040007,
 277	0xf01a, 0xffffffff, 0x00060005,
 278	0xf01b, 0xffffffff, 0x00090008,
 279	0xf01c, 0xffffffff, 0x00010000,
 280	0xf01d, 0xffffffff, 0x00030002,
 281	0xf01e, 0xffffffff, 0x00040007,
 282	0xf01f, 0xffffffff, 0x00060005,
 283	0xf020, 0xffffffff, 0x00090008,
 284	0xf021, 0xffffffff, 0x00010000,
 285	0xf022, 0xffffffff, 0x00030002,
 286	0xf023, 0xffffffff, 0x00040007,
 287	0xf024, 0xffffffff, 0x00060005,
 288	0xf025, 0xffffffff, 0x00090008,
 289	0xf026, 0xffffffff, 0x00010000,
 290	0xf027, 0xffffffff, 0x00030002,
 291	0xf028, 0xffffffff, 0x00040007,
 292	0xf029, 0xffffffff, 0x00060005,
 293	0xf02a, 0xffffffff, 0x00090008,
 294	0xf000, 0xffffffff, 0x96e00200,
 295	0x21c2, 0xffffffff, 0x00900100,
 296	0x3109, 0xffffffff, 0x0020003f,
 297	0xe, 0xffffffff, 0x0140001c,
 298	0xf, 0x000f0000, 0x000f0000,
 299	0x88, 0xffffffff, 0xc060000c,
 300	0x89, 0xc0000fff, 0x00000100,
 301	0x3e4, 0xffffffff, 0x00000100,
 302	0x3e6, 0x00000101, 0x00000000,
 303	0x82a, 0xffffffff, 0x00000104,
 304	0x1579, 0xff000fff, 0x00000100,
 305	0xc33, 0xc0000fff, 0x00000104,
 306	0x3079, 0x00000001, 0x00000001,
 307	0x3403, 0xff000ff0, 0x00000100,
 308	0x3603, 0xff000ff0, 0x00000100
 309};
 310
 311static const u32 spectre_golden_spm_registers[] =
 312{
 313	0xc200, 0xe0ffffff, 0xe0000000
 314};
 315
 316static const u32 spectre_golden_common_registers[] =
 317{
 318	0x31dc, 0xffffffff, 0x00000800,
 319	0x31dd, 0xffffffff, 0x00000800,
 320	0x31e6, 0xffffffff, 0x00007fbf,
 321	0x31e7, 0xffffffff, 0x00007faf
 322};
 323
 324static const u32 spectre_golden_registers[] =
 325{
 326	0xf000, 0xffff1fff, 0x96940200,
 327	0xf003, 0xffff0001, 0xff000000,
 328	0xf080, 0xfffc0fff, 0x00000100,
 329	0x1bb6, 0x00010101, 0x00010000,
 330	0x260d, 0xf00fffff, 0x00000400,
 331	0x260e, 0xfffffffc, 0x00020200,
 332	0x16ec, 0x000000f0, 0x00000070,
 333	0x16f0, 0xf0311fff, 0x80300000,
 334	0x263e, 0x73773777, 0x12010001,
 335	0x26df, 0x00ff0000, 0x00fc0000,
 336	0xbd2, 0x73773777, 0x12010001,
 337	0x2285, 0xf000003f, 0x00000007,
 338	0x22c9, 0xffffffff, 0x00ffffff,
 339	0xa0d4, 0x3f3f3fff, 0x00000082,
 340	0xa0d5, 0x0000003f, 0x00000000,
 341	0xf9e, 0x00000001, 0x00000002,
 342	0x244f, 0xffff03df, 0x00000004,
 343	0x31da, 0x00000008, 0x00000008,
 344	0x2300, 0x000008ff, 0x00000800,
 345	0x2542, 0x00010000, 0x00010000,
 346	0x2b03, 0xffffffff, 0x54763210,
 347	0x853e, 0x01ff01ff, 0x00000002,
 348	0x8526, 0x007ff800, 0x00200000,
 349	0x8057, 0xffffffff, 0x00000f40,
 350	0xc24d, 0xffffffff, 0x00000001
 351};
 352
 353static const u32 spectre_mgcg_cgcg_init[] =
 354{
 355	0x3108, 0xffffffff, 0xfffffffc,
 356	0xc200, 0xffffffff, 0xe0000000,
 357	0xf0a8, 0xffffffff, 0x00000100,
 358	0xf082, 0xffffffff, 0x00000100,
 359	0xf0b0, 0xffffffff, 0x00000100,
 360	0xf0b2, 0xffffffff, 0x00000100,
 361	0xf0b1, 0xffffffff, 0x00000100,
 362	0x1579, 0xffffffff, 0x00600100,
 363	0xf0a0, 0xffffffff, 0x00000100,
 364	0xf085, 0xffffffff, 0x06000100,
 365	0xf088, 0xffffffff, 0x00000100,
 366	0xf086, 0xffffffff, 0x06000100,
 367	0xf081, 0xffffffff, 0x00000100,
 368	0xf0b8, 0xffffffff, 0x00000100,
 369	0xf089, 0xffffffff, 0x00000100,
 370	0xf080, 0xffffffff, 0x00000100,
 371	0xf08c, 0xffffffff, 0x00000100,
 372	0xf08d, 0xffffffff, 0x00000100,
 373	0xf094, 0xffffffff, 0x00000100,
 374	0xf095, 0xffffffff, 0x00000100,
 375	0xf096, 0xffffffff, 0x00000100,
 376	0xf097, 0xffffffff, 0x00000100,
 377	0xf098, 0xffffffff, 0x00000100,
 378	0xf09f, 0xffffffff, 0x00000100,
 379	0xf09e, 0xffffffff, 0x00000100,
 380	0xf084, 0xffffffff, 0x06000100,
 381	0xf0a4, 0xffffffff, 0x00000100,
 382	0xf09d, 0xffffffff, 0x00000100,
 383	0xf0ad, 0xffffffff, 0x00000100,
 384	0xf0ac, 0xffffffff, 0x00000100,
 385	0xf09c, 0xffffffff, 0x00000100,
 386	0xc200, 0xffffffff, 0xe0000000,
 387	0xf008, 0xffffffff, 0x00010000,
 388	0xf009, 0xffffffff, 0x00030002,
 389	0xf00a, 0xffffffff, 0x00040007,
 390	0xf00b, 0xffffffff, 0x00060005,
 391	0xf00c, 0xffffffff, 0x00090008,
 392	0xf00d, 0xffffffff, 0x00010000,
 393	0xf00e, 0xffffffff, 0x00030002,
 394	0xf00f, 0xffffffff, 0x00040007,
 395	0xf010, 0xffffffff, 0x00060005,
 396	0xf011, 0xffffffff, 0x00090008,
 397	0xf012, 0xffffffff, 0x00010000,
 398	0xf013, 0xffffffff, 0x00030002,
 399	0xf014, 0xffffffff, 0x00040007,
 400	0xf015, 0xffffffff, 0x00060005,
 401	0xf016, 0xffffffff, 0x00090008,
 402	0xf017, 0xffffffff, 0x00010000,
 403	0xf018, 0xffffffff, 0x00030002,
 404	0xf019, 0xffffffff, 0x00040007,
 405	0xf01a, 0xffffffff, 0x00060005,
 406	0xf01b, 0xffffffff, 0x00090008,
 407	0xf01c, 0xffffffff, 0x00010000,
 408	0xf01d, 0xffffffff, 0x00030002,
 409	0xf01e, 0xffffffff, 0x00040007,
 410	0xf01f, 0xffffffff, 0x00060005,
 411	0xf020, 0xffffffff, 0x00090008,
 412	0xf021, 0xffffffff, 0x00010000,
 413	0xf022, 0xffffffff, 0x00030002,
 414	0xf023, 0xffffffff, 0x00040007,
 415	0xf024, 0xffffffff, 0x00060005,
 416	0xf025, 0xffffffff, 0x00090008,
 417	0xf026, 0xffffffff, 0x00010000,
 418	0xf027, 0xffffffff, 0x00030002,
 419	0xf028, 0xffffffff, 0x00040007,
 420	0xf029, 0xffffffff, 0x00060005,
 421	0xf02a, 0xffffffff, 0x00090008,
 422	0xf02b, 0xffffffff, 0x00010000,
 423	0xf02c, 0xffffffff, 0x00030002,
 424	0xf02d, 0xffffffff, 0x00040007,
 425	0xf02e, 0xffffffff, 0x00060005,
 426	0xf02f, 0xffffffff, 0x00090008,
 427	0xf000, 0xffffffff, 0x96e00200,
 428	0x21c2, 0xffffffff, 0x00900100,
 429	0x3109, 0xffffffff, 0x0020003f,
 430	0xe, 0xffffffff, 0x0140001c,
 431	0xf, 0x000f0000, 0x000f0000,
 432	0x88, 0xffffffff, 0xc060000c,
 433	0x89, 0xc0000fff, 0x00000100,
 434	0x3e4, 0xffffffff, 0x00000100,
 435	0x3e6, 0x00000101, 0x00000000,
 436	0x82a, 0xffffffff, 0x00000104,
 437	0x1579, 0xff000fff, 0x00000100,
 438	0xc33, 0xc0000fff, 0x00000104,
 439	0x3079, 0x00000001, 0x00000001,
 440	0x3403, 0xff000ff0, 0x00000100,
 441	0x3603, 0xff000ff0, 0x00000100
 442};
 443
 444static const u32 kalindi_golden_spm_registers[] =
 445{
 446	0xc200, 0xe0ffffff, 0xe0000000
 447};
 448
 449static const u32 kalindi_golden_common_registers[] =
 450{
 451	0x31dc, 0xffffffff, 0x00000800,
 452	0x31dd, 0xffffffff, 0x00000800,
 453	0x31e6, 0xffffffff, 0x00007fbf,
 454	0x31e7, 0xffffffff, 0x00007faf
 455};
 456
 457static const u32 kalindi_golden_registers[] =
 458{
 459	0xf000, 0xffffdfff, 0x6e944040,
 460	0x1579, 0xff607fff, 0xfc000100,
 461	0xf088, 0xff000fff, 0x00000100,
 462	0xf089, 0xff000fff, 0x00000100,
 463	0xf080, 0xfffc0fff, 0x00000100,
 464	0x1bb6, 0x00010101, 0x00010000,
 465	0x260c, 0xffffffff, 0x00000000,
 466	0x260d, 0xf00fffff, 0x00000400,
 467	0x16ec, 0x000000f0, 0x00000070,
 468	0x16f0, 0xf0311fff, 0x80300000,
 469	0x263e, 0x73773777, 0x12010001,
 470	0x263f, 0xffffffff, 0x00000010,
 471	0x26df, 0x00ff0000, 0x00fc0000,
 472	0x200c, 0x00001f0f, 0x0000100a,
 473	0xbd2, 0x73773777, 0x12010001,
 474	0x902, 0x000fffff, 0x000c007f,
 475	0x2285, 0xf000003f, 0x00000007,
 476	0x22c9, 0x3fff3fff, 0x00ffcfff,
 477	0xc281, 0x0000ff0f, 0x00000000,
 478	0xa293, 0x07ffffff, 0x06000000,
 479	0x136, 0x00000fff, 0x00000100,
 480	0xf9e, 0x00000001, 0x00000002,
 481	0x31da, 0x00000008, 0x00000008,
 482	0x2300, 0x000000ff, 0x00000003,
 483	0x853e, 0x01ff01ff, 0x00000002,
 484	0x8526, 0x007ff800, 0x00200000,
 485	0x8057, 0xffffffff, 0x00000f40,
 486	0x2231, 0x001f3ae3, 0x00000082,
 487	0x2235, 0x0000001f, 0x00000010,
 488	0xc24d, 0xffffffff, 0x00000000
 489};
 490
 491static const u32 kalindi_mgcg_cgcg_init[] =
 492{
 493	0x3108, 0xffffffff, 0xfffffffc,
 494	0xc200, 0xffffffff, 0xe0000000,
 495	0xf0a8, 0xffffffff, 0x00000100,
 496	0xf082, 0xffffffff, 0x00000100,
 497	0xf0b0, 0xffffffff, 0x00000100,
 498	0xf0b2, 0xffffffff, 0x00000100,
 499	0xf0b1, 0xffffffff, 0x00000100,
 500	0x1579, 0xffffffff, 0x00600100,
 501	0xf0a0, 0xffffffff, 0x00000100,
 502	0xf085, 0xffffffff, 0x06000100,
 503	0xf088, 0xffffffff, 0x00000100,
 504	0xf086, 0xffffffff, 0x06000100,
 505	0xf081, 0xffffffff, 0x00000100,
 506	0xf0b8, 0xffffffff, 0x00000100,
 507	0xf089, 0xffffffff, 0x00000100,
 508	0xf080, 0xffffffff, 0x00000100,
 509	0xf08c, 0xffffffff, 0x00000100,
 510	0xf08d, 0xffffffff, 0x00000100,
 511	0xf094, 0xffffffff, 0x00000100,
 512	0xf095, 0xffffffff, 0x00000100,
 513	0xf096, 0xffffffff, 0x00000100,
 514	0xf097, 0xffffffff, 0x00000100,
 515	0xf098, 0xffffffff, 0x00000100,
 516	0xf09f, 0xffffffff, 0x00000100,
 517	0xf09e, 0xffffffff, 0x00000100,
 518	0xf084, 0xffffffff, 0x06000100,
 519	0xf0a4, 0xffffffff, 0x00000100,
 520	0xf09d, 0xffffffff, 0x00000100,
 521	0xf0ad, 0xffffffff, 0x00000100,
 522	0xf0ac, 0xffffffff, 0x00000100,
 523	0xf09c, 0xffffffff, 0x00000100,
 524	0xc200, 0xffffffff, 0xe0000000,
 525	0xf008, 0xffffffff, 0x00010000,
 526	0xf009, 0xffffffff, 0x00030002,
 527	0xf00a, 0xffffffff, 0x00040007,
 528	0xf00b, 0xffffffff, 0x00060005,
 529	0xf00c, 0xffffffff, 0x00090008,
 530	0xf00d, 0xffffffff, 0x00010000,
 531	0xf00e, 0xffffffff, 0x00030002,
 532	0xf00f, 0xffffffff, 0x00040007,
 533	0xf010, 0xffffffff, 0x00060005,
 534	0xf011, 0xffffffff, 0x00090008,
 535	0xf000, 0xffffffff, 0x96e00200,
 536	0x21c2, 0xffffffff, 0x00900100,
 537	0x3109, 0xffffffff, 0x0020003f,
 538	0xe, 0xffffffff, 0x0140001c,
 539	0xf, 0x000f0000, 0x000f0000,
 540	0x88, 0xffffffff, 0xc060000c,
 541	0x89, 0xc0000fff, 0x00000100,
 542	0x82a, 0xffffffff, 0x00000104,
 543	0x1579, 0xff000fff, 0x00000100,
 544	0xc33, 0xc0000fff, 0x00000104,
 545	0x3079, 0x00000001, 0x00000001,
 546	0x3403, 0xff000ff0, 0x00000100,
 547	0x3603, 0xff000ff0, 0x00000100
 548};
 549
 550static const u32 hawaii_golden_spm_registers[] =
 551{
 552	0xc200, 0xe0ffffff, 0xe0000000
 553};
 554
 555static const u32 hawaii_golden_common_registers[] =
 556{
 557	0xc200, 0xffffffff, 0xe0000000,
 558	0xa0d4, 0xffffffff, 0x3a00161a,
 559	0xa0d5, 0xffffffff, 0x0000002e,
 560	0x2684, 0xffffffff, 0x00018208,
 561	0x263e, 0xffffffff, 0x12011003
 562};
 563
 564static const u32 hawaii_golden_registers[] =
 565{
 566	0xcd5, 0x00000333, 0x00000333,
 567	0x2684, 0x00010000, 0x00058208,
 568	0x260c, 0xffffffff, 0x00000000,
 569	0x260d, 0xf00fffff, 0x00000400,
 570	0x260e, 0x0002021c, 0x00020200,
 571	0x31e, 0x00000080, 0x00000000,
 572	0x16ec, 0x000000f0, 0x00000070,
 573	0x16f0, 0xf0311fff, 0x80300000,
 574	0xd43, 0x00810000, 0x408af000,
 575	0x1c0c, 0x31000111, 0x00000011,
 576	0xbd2, 0x73773777, 0x12010001,
 577	0x848, 0x0000007f, 0x0000001b,
 578	0x877, 0x00007fb6, 0x00002191,
 579	0xd8a, 0x0000003f, 0x0000000a,
 580	0xd8b, 0x0000003f, 0x0000000a,
 581	0xab9, 0x00073ffe, 0x000022a2,
 582	0x903, 0x000007ff, 0x00000000,
 583	0x22fc, 0x00002001, 0x00000001,
 584	0x22c9, 0xffffffff, 0x00ffffff,
 585	0xc281, 0x0000ff0f, 0x00000000,
 586	0xa293, 0x07ffffff, 0x06000000,
 587	0xf9e, 0x00000001, 0x00000002,
 588	0x31da, 0x00000008, 0x00000008,
 589	0x31dc, 0x00000f00, 0x00000800,
 590	0x31dd, 0x00000f00, 0x00000800,
 591	0x31e6, 0x00ffffff, 0x00ff7fbf,
 592	0x31e7, 0x00ffffff, 0x00ff7faf,
 593	0x2300, 0x000000ff, 0x00000800,
 594	0x390, 0x00001fff, 0x00001fff,
 595	0x2418, 0x0000007f, 0x00000020,
 596	0x2542, 0x00010000, 0x00010000,
 597	0x2b80, 0x00100000, 0x000ff07c,
 598	0x2b05, 0x000003ff, 0x0000000f,
 599	0x2b04, 0xffffffff, 0x7564fdec,
 600	0x2b03, 0xffffffff, 0x3120b9a8,
 601	0x2b02, 0x20000000, 0x0f9c0000
 602};
 603
 604static const u32 hawaii_mgcg_cgcg_init[] =
 605{
 606	0x3108, 0xffffffff, 0xfffffffd,
 607	0xc200, 0xffffffff, 0xe0000000,
 608	0xf0a8, 0xffffffff, 0x00000100,
 609	0xf082, 0xffffffff, 0x00000100,
 610	0xf0b0, 0xffffffff, 0x00000100,
 611	0xf0b2, 0xffffffff, 0x00000100,
 612	0xf0b1, 0xffffffff, 0x00000100,
 613	0x1579, 0xffffffff, 0x00200100,
 614	0xf0a0, 0xffffffff, 0x00000100,
 615	0xf085, 0xffffffff, 0x06000100,
 616	0xf088, 0xffffffff, 0x00000100,
 617	0xf086, 0xffffffff, 0x06000100,
 618	0xf081, 0xffffffff, 0x00000100,
 619	0xf0b8, 0xffffffff, 0x00000100,
 620	0xf089, 0xffffffff, 0x00000100,
 621	0xf080, 0xffffffff, 0x00000100,
 622	0xf08c, 0xffffffff, 0x00000100,
 623	0xf08d, 0xffffffff, 0x00000100,
 624	0xf094, 0xffffffff, 0x00000100,
 625	0xf095, 0xffffffff, 0x00000100,
 626	0xf096, 0xffffffff, 0x00000100,
 627	0xf097, 0xffffffff, 0x00000100,
 628	0xf098, 0xffffffff, 0x00000100,
 629	0xf09f, 0xffffffff, 0x00000100,
 630	0xf09e, 0xffffffff, 0x00000100,
 631	0xf084, 0xffffffff, 0x06000100,
 632	0xf0a4, 0xffffffff, 0x00000100,
 633	0xf09d, 0xffffffff, 0x00000100,
 634	0xf0ad, 0xffffffff, 0x00000100,
 635	0xf0ac, 0xffffffff, 0x00000100,
 636	0xf09c, 0xffffffff, 0x00000100,
 637	0xc200, 0xffffffff, 0xe0000000,
 638	0xf008, 0xffffffff, 0x00010000,
 639	0xf009, 0xffffffff, 0x00030002,
 640	0xf00a, 0xffffffff, 0x00040007,
 641	0xf00b, 0xffffffff, 0x00060005,
 642	0xf00c, 0xffffffff, 0x00090008,
 643	0xf00d, 0xffffffff, 0x00010000,
 644	0xf00e, 0xffffffff, 0x00030002,
 645	0xf00f, 0xffffffff, 0x00040007,
 646	0xf010, 0xffffffff, 0x00060005,
 647	0xf011, 0xffffffff, 0x00090008,
 648	0xf012, 0xffffffff, 0x00010000,
 649	0xf013, 0xffffffff, 0x00030002,
 650	0xf014, 0xffffffff, 0x00040007,
 651	0xf015, 0xffffffff, 0x00060005,
 652	0xf016, 0xffffffff, 0x00090008,
 653	0xf017, 0xffffffff, 0x00010000,
 654	0xf018, 0xffffffff, 0x00030002,
 655	0xf019, 0xffffffff, 0x00040007,
 656	0xf01a, 0xffffffff, 0x00060005,
 657	0xf01b, 0xffffffff, 0x00090008,
 658	0xf01c, 0xffffffff, 0x00010000,
 659	0xf01d, 0xffffffff, 0x00030002,
 660	0xf01e, 0xffffffff, 0x00040007,
 661	0xf01f, 0xffffffff, 0x00060005,
 662	0xf020, 0xffffffff, 0x00090008,
 663	0xf021, 0xffffffff, 0x00010000,
 664	0xf022, 0xffffffff, 0x00030002,
 665	0xf023, 0xffffffff, 0x00040007,
 666	0xf024, 0xffffffff, 0x00060005,
 667	0xf025, 0xffffffff, 0x00090008,
 668	0xf026, 0xffffffff, 0x00010000,
 669	0xf027, 0xffffffff, 0x00030002,
 670	0xf028, 0xffffffff, 0x00040007,
 671	0xf029, 0xffffffff, 0x00060005,
 672	0xf02a, 0xffffffff, 0x00090008,
 673	0xf02b, 0xffffffff, 0x00010000,
 674	0xf02c, 0xffffffff, 0x00030002,
 675	0xf02d, 0xffffffff, 0x00040007,
 676	0xf02e, 0xffffffff, 0x00060005,
 677	0xf02f, 0xffffffff, 0x00090008,
 678	0xf030, 0xffffffff, 0x00010000,
 679	0xf031, 0xffffffff, 0x00030002,
 680	0xf032, 0xffffffff, 0x00040007,
 681	0xf033, 0xffffffff, 0x00060005,
 682	0xf034, 0xffffffff, 0x00090008,
 683	0xf035, 0xffffffff, 0x00010000,
 684	0xf036, 0xffffffff, 0x00030002,
 685	0xf037, 0xffffffff, 0x00040007,
 686	0xf038, 0xffffffff, 0x00060005,
 687	0xf039, 0xffffffff, 0x00090008,
 688	0xf03a, 0xffffffff, 0x00010000,
 689	0xf03b, 0xffffffff, 0x00030002,
 690	0xf03c, 0xffffffff, 0x00040007,
 691	0xf03d, 0xffffffff, 0x00060005,
 692	0xf03e, 0xffffffff, 0x00090008,
 693	0x30c6, 0xffffffff, 0x00020200,
 694	0xcd4, 0xffffffff, 0x00000200,
 695	0x570, 0xffffffff, 0x00000400,
 696	0x157a, 0xffffffff, 0x00000000,
 697	0xbd4, 0xffffffff, 0x00000902,
 698	0xf000, 0xffffffff, 0x96940200,
 699	0x21c2, 0xffffffff, 0x00900100,
 700	0x3109, 0xffffffff, 0x0020003f,
 701	0xe, 0xffffffff, 0x0140001c,
 702	0xf, 0x000f0000, 0x000f0000,
 703	0x88, 0xffffffff, 0xc060000c,
 704	0x89, 0xc0000fff, 0x00000100,
 705	0x3e4, 0xffffffff, 0x00000100,
 706	0x3e6, 0x00000101, 0x00000000,
 707	0x82a, 0xffffffff, 0x00000104,
 708	0x1579, 0xff000fff, 0x00000100,
 709	0xc33, 0xc0000fff, 0x00000104,
 710	0x3079, 0x00000001, 0x00000001,
 711	0x3403, 0xff000ff0, 0x00000100,
 712	0x3603, 0xff000ff0, 0x00000100
 713};
 714
 715static const u32 godavari_golden_registers[] =
 716{
 717	0x1579, 0xff607fff, 0xfc000100,
 718	0x1bb6, 0x00010101, 0x00010000,
 719	0x260c, 0xffffffff, 0x00000000,
 720	0x260c0, 0xf00fffff, 0x00000400,
 721	0x184c, 0xffffffff, 0x00010000,
 722	0x16ec, 0x000000f0, 0x00000070,
 723	0x16f0, 0xf0311fff, 0x80300000,
 724	0x263e, 0x73773777, 0x12010001,
 725	0x263f, 0xffffffff, 0x00000010,
 726	0x200c, 0x00001f0f, 0x0000100a,
 727	0xbd2, 0x73773777, 0x12010001,
 728	0x902, 0x000fffff, 0x000c007f,
 729	0x2285, 0xf000003f, 0x00000007,
 730	0x22c9, 0xffffffff, 0x00ff0fff,
 731	0xc281, 0x0000ff0f, 0x00000000,
 732	0xa293, 0x07ffffff, 0x06000000,
 733	0x136, 0x00000fff, 0x00000100,
 734	0x3405, 0x00010000, 0x00810001,
 735	0x3605, 0x00010000, 0x00810001,
 736	0xf9e, 0x00000001, 0x00000002,
 737	0x31da, 0x00000008, 0x00000008,
 738	0x31dc, 0x00000f00, 0x00000800,
 739	0x31dd, 0x00000f00, 0x00000800,
 740	0x31e6, 0x00ffffff, 0x00ff7fbf,
 741	0x31e7, 0x00ffffff, 0x00ff7faf,
 742	0x2300, 0x000000ff, 0x00000001,
 743	0x853e, 0x01ff01ff, 0x00000002,
 744	0x8526, 0x007ff800, 0x00200000,
 745	0x8057, 0xffffffff, 0x00000f40,
 746	0x2231, 0x001f3ae3, 0x00000082,
 747	0x2235, 0x0000001f, 0x00000010,
 748	0xc24d, 0xffffffff, 0x00000000
 749};
 750
 751static void cik_init_golden_registers(struct amdgpu_device *adev)
 752{
 753	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 754	mutex_lock(&adev->grbm_idx_mutex);
 755
 756	switch (adev->asic_type) {
 757	case CHIP_BONAIRE:
 758		amdgpu_device_program_register_sequence(adev,
 759							bonaire_mgcg_cgcg_init,
 760							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 761		amdgpu_device_program_register_sequence(adev,
 762							bonaire_golden_registers,
 763							ARRAY_SIZE(bonaire_golden_registers));
 764		amdgpu_device_program_register_sequence(adev,
 765							bonaire_golden_common_registers,
 766							ARRAY_SIZE(bonaire_golden_common_registers));
 767		amdgpu_device_program_register_sequence(adev,
 768							bonaire_golden_spm_registers,
 769							ARRAY_SIZE(bonaire_golden_spm_registers));
 770		break;
 771	case CHIP_KABINI:
 772		amdgpu_device_program_register_sequence(adev,
 773							kalindi_mgcg_cgcg_init,
 774							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 775		amdgpu_device_program_register_sequence(adev,
 776							kalindi_golden_registers,
 777							ARRAY_SIZE(kalindi_golden_registers));
 778		amdgpu_device_program_register_sequence(adev,
 779							kalindi_golden_common_registers,
 780							ARRAY_SIZE(kalindi_golden_common_registers));
 781		amdgpu_device_program_register_sequence(adev,
 782							kalindi_golden_spm_registers,
 783							ARRAY_SIZE(kalindi_golden_spm_registers));
 784		break;
 785	case CHIP_MULLINS:
 786		amdgpu_device_program_register_sequence(adev,
 787							kalindi_mgcg_cgcg_init,
 788							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 789		amdgpu_device_program_register_sequence(adev,
 790							godavari_golden_registers,
 791							ARRAY_SIZE(godavari_golden_registers));
 792		amdgpu_device_program_register_sequence(adev,
 793							kalindi_golden_common_registers,
 794							ARRAY_SIZE(kalindi_golden_common_registers));
 795		amdgpu_device_program_register_sequence(adev,
 796							kalindi_golden_spm_registers,
 797							ARRAY_SIZE(kalindi_golden_spm_registers));
 798		break;
 799	case CHIP_KAVERI:
 800		amdgpu_device_program_register_sequence(adev,
 801							spectre_mgcg_cgcg_init,
 802							ARRAY_SIZE(spectre_mgcg_cgcg_init));
 803		amdgpu_device_program_register_sequence(adev,
 804							spectre_golden_registers,
 805							ARRAY_SIZE(spectre_golden_registers));
 806		amdgpu_device_program_register_sequence(adev,
 807							spectre_golden_common_registers,
 808							ARRAY_SIZE(spectre_golden_common_registers));
 809		amdgpu_device_program_register_sequence(adev,
 810							spectre_golden_spm_registers,
 811							ARRAY_SIZE(spectre_golden_spm_registers));
 812		break;
 813	case CHIP_HAWAII:
 814		amdgpu_device_program_register_sequence(adev,
 815							hawaii_mgcg_cgcg_init,
 816							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 817		amdgpu_device_program_register_sequence(adev,
 818							hawaii_golden_registers,
 819							ARRAY_SIZE(hawaii_golden_registers));
 820		amdgpu_device_program_register_sequence(adev,
 821							hawaii_golden_common_registers,
 822							ARRAY_SIZE(hawaii_golden_common_registers));
 823		amdgpu_device_program_register_sequence(adev,
 824							hawaii_golden_spm_registers,
 825							ARRAY_SIZE(hawaii_golden_spm_registers));
 826		break;
 827	default:
 828		break;
 829	}
 830	mutex_unlock(&adev->grbm_idx_mutex);
 831}
 832
 833/**
 834 * cik_get_xclk - get the xclk
 835 *
 836 * @adev: amdgpu_device pointer
 837 *
 838 * Returns the reference clock used by the gfx engine
 839 * (CIK).
 840 */
 841static u32 cik_get_xclk(struct amdgpu_device *adev)
 842{
 843	u32 reference_clock = adev->clock.spll.reference_freq;
 844
 845	if (adev->flags & AMD_IS_APU) {
 846		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 847			return reference_clock / 2;
 848	} else {
 849		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 850			return reference_clock / 4;
 851	}
 852	return reference_clock;
 853}
 854
 855/**
 856 * cik_srbm_select - select specific register instances
 857 *
 858 * @adev: amdgpu_device pointer
 859 * @me: selected ME (micro engine)
 860 * @pipe: pipe
 861 * @queue: queue
 862 * @vmid: VMID
 863 *
 864 * Switches the currently active registers instances.  Some
 865 * registers are instanced per VMID, others are instanced per
 866 * me/pipe/queue combination.
 867 */
 868void cik_srbm_select(struct amdgpu_device *adev,
 869		     u32 me, u32 pipe, u32 queue, u32 vmid)
 870{
 871	u32 srbm_gfx_cntl =
 872		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 873		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 874		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 875		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 876	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 877}
 878
 879static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 880{
 881	uint32_t tmp;
 882
 883	tmp = RREG32(mmCONFIG_CNTL);
 884	if (!state)
 885		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 886	else
 887		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 888	WREG32(mmCONFIG_CNTL, tmp);
 889}
 890
 891static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 892{
 893	u32 bus_cntl;
 894	u32 d1vga_control = 0;
 895	u32 d2vga_control = 0;
 896	u32 vga_render_control = 0;
 897	u32 rom_cntl;
 898	bool r;
 899
 900	bus_cntl = RREG32(mmBUS_CNTL);
 901	if (adev->mode_info.num_crtc) {
 902		d1vga_control = RREG32(mmD1VGA_CONTROL);
 903		d2vga_control = RREG32(mmD2VGA_CONTROL);
 904		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 905	}
 906	rom_cntl = RREG32_SMC(ixROM_CNTL);
 907
 908	/* enable the rom */
 909	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 910	if (adev->mode_info.num_crtc) {
 911		/* Disable VGA mode */
 912		WREG32(mmD1VGA_CONTROL,
 913		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 914					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 915		WREG32(mmD2VGA_CONTROL,
 916		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 917					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 918		WREG32(mmVGA_RENDER_CONTROL,
 919		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 920	}
 921	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 922
 923	r = amdgpu_read_bios(adev);
 924
 925	/* restore regs */
 926	WREG32(mmBUS_CNTL, bus_cntl);
 927	if (adev->mode_info.num_crtc) {
 928		WREG32(mmD1VGA_CONTROL, d1vga_control);
 929		WREG32(mmD2VGA_CONTROL, d2vga_control);
 930		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 931	}
 932	WREG32_SMC(ixROM_CNTL, rom_cntl);
 933	return r;
 934}
 935
 936static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 937				   u8 *bios, u32 length_bytes)
 938{
 939	u32 *dw_ptr;
 940	unsigned long flags;
 941	u32 i, length_dw;
 942
 943	if (bios == NULL)
 944		return false;
 945	if (length_bytes == 0)
 946		return false;
 947	/* APU vbios image is part of sbios image */
 948	if (adev->flags & AMD_IS_APU)
 949		return false;
 950
 951	dw_ptr = (u32 *)bios;
 952	length_dw = ALIGN(length_bytes, 4) / 4;
 953	/* take the smc lock since we are using the smc index */
 954	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 955	/* set rom index to 0 */
 956	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 957	WREG32(mmSMC_IND_DATA_0, 0);
 958	/* set index to data for continous read */
 959	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 960	for (i = 0; i < length_dw; i++)
 961		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 962	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 963
 964	return true;
 965}
 966
 967static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 968	{mmGRBM_STATUS},
 969	{mmGB_ADDR_CONFIG},
 970	{mmMC_ARB_RAMCFG},
 971	{mmGB_TILE_MODE0},
 972	{mmGB_TILE_MODE1},
 973	{mmGB_TILE_MODE2},
 974	{mmGB_TILE_MODE3},
 975	{mmGB_TILE_MODE4},
 976	{mmGB_TILE_MODE5},
 977	{mmGB_TILE_MODE6},
 978	{mmGB_TILE_MODE7},
 979	{mmGB_TILE_MODE8},
 980	{mmGB_TILE_MODE9},
 981	{mmGB_TILE_MODE10},
 982	{mmGB_TILE_MODE11},
 983	{mmGB_TILE_MODE12},
 984	{mmGB_TILE_MODE13},
 985	{mmGB_TILE_MODE14},
 986	{mmGB_TILE_MODE15},
 987	{mmGB_TILE_MODE16},
 988	{mmGB_TILE_MODE17},
 989	{mmGB_TILE_MODE18},
 990	{mmGB_TILE_MODE19},
 991	{mmGB_TILE_MODE20},
 992	{mmGB_TILE_MODE21},
 993	{mmGB_TILE_MODE22},
 994	{mmGB_TILE_MODE23},
 995	{mmGB_TILE_MODE24},
 996	{mmGB_TILE_MODE25},
 997	{mmGB_TILE_MODE26},
 998	{mmGB_TILE_MODE27},
 999	{mmGB_TILE_MODE28},
1000	{mmGB_TILE_MODE29},
1001	{mmGB_TILE_MODE30},
1002	{mmGB_TILE_MODE31},
1003	{mmGB_MACROTILE_MODE0},
1004	{mmGB_MACROTILE_MODE1},
1005	{mmGB_MACROTILE_MODE2},
1006	{mmGB_MACROTILE_MODE3},
1007	{mmGB_MACROTILE_MODE4},
1008	{mmGB_MACROTILE_MODE5},
1009	{mmGB_MACROTILE_MODE6},
1010	{mmGB_MACROTILE_MODE7},
1011	{mmGB_MACROTILE_MODE8},
1012	{mmGB_MACROTILE_MODE9},
1013	{mmGB_MACROTILE_MODE10},
1014	{mmGB_MACROTILE_MODE11},
1015	{mmGB_MACROTILE_MODE12},
1016	{mmGB_MACROTILE_MODE13},
1017	{mmGB_MACROTILE_MODE14},
1018	{mmGB_MACROTILE_MODE15},
1019	{mmCC_RB_BACKEND_DISABLE, true},
1020	{mmGC_USER_RB_BACKEND_DISABLE, true},
1021	{mmGB_BACKEND_MAP, false},
1022	{mmPA_SC_RASTER_CONFIG, true},
1023	{mmPA_SC_RASTER_CONFIG_1, true},
1024};
1025
1026
1027static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1028				       bool indexed, u32 se_num,
1029				       u32 sh_num, u32 reg_offset)
1030{
1031	if (indexed) {
1032		uint32_t val;
1033		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1034		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1035
1036		switch (reg_offset) {
1037		case mmCC_RB_BACKEND_DISABLE:
1038			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1039		case mmGC_USER_RB_BACKEND_DISABLE:
1040			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1041		case mmPA_SC_RASTER_CONFIG:
1042			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1043		case mmPA_SC_RASTER_CONFIG_1:
1044			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1045		}
1046
1047		mutex_lock(&adev->grbm_idx_mutex);
1048		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1049			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1050
1051		val = RREG32(reg_offset);
1052
1053		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1054			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1055		mutex_unlock(&adev->grbm_idx_mutex);
1056		return val;
1057	} else {
1058		unsigned idx;
1059
1060		switch (reg_offset) {
1061		case mmGB_ADDR_CONFIG:
1062			return adev->gfx.config.gb_addr_config;
1063		case mmMC_ARB_RAMCFG:
1064			return adev->gfx.config.mc_arb_ramcfg;
1065		case mmGB_TILE_MODE0:
1066		case mmGB_TILE_MODE1:
1067		case mmGB_TILE_MODE2:
1068		case mmGB_TILE_MODE3:
1069		case mmGB_TILE_MODE4:
1070		case mmGB_TILE_MODE5:
1071		case mmGB_TILE_MODE6:
1072		case mmGB_TILE_MODE7:
1073		case mmGB_TILE_MODE8:
1074		case mmGB_TILE_MODE9:
1075		case mmGB_TILE_MODE10:
1076		case mmGB_TILE_MODE11:
1077		case mmGB_TILE_MODE12:
1078		case mmGB_TILE_MODE13:
1079		case mmGB_TILE_MODE14:
1080		case mmGB_TILE_MODE15:
1081		case mmGB_TILE_MODE16:
1082		case mmGB_TILE_MODE17:
1083		case mmGB_TILE_MODE18:
1084		case mmGB_TILE_MODE19:
1085		case mmGB_TILE_MODE20:
1086		case mmGB_TILE_MODE21:
1087		case mmGB_TILE_MODE22:
1088		case mmGB_TILE_MODE23:
1089		case mmGB_TILE_MODE24:
1090		case mmGB_TILE_MODE25:
1091		case mmGB_TILE_MODE26:
1092		case mmGB_TILE_MODE27:
1093		case mmGB_TILE_MODE28:
1094		case mmGB_TILE_MODE29:
1095		case mmGB_TILE_MODE30:
1096		case mmGB_TILE_MODE31:
1097			idx = (reg_offset - mmGB_TILE_MODE0);
1098			return adev->gfx.config.tile_mode_array[idx];
1099		case mmGB_MACROTILE_MODE0:
1100		case mmGB_MACROTILE_MODE1:
1101		case mmGB_MACROTILE_MODE2:
1102		case mmGB_MACROTILE_MODE3:
1103		case mmGB_MACROTILE_MODE4:
1104		case mmGB_MACROTILE_MODE5:
1105		case mmGB_MACROTILE_MODE6:
1106		case mmGB_MACROTILE_MODE7:
1107		case mmGB_MACROTILE_MODE8:
1108		case mmGB_MACROTILE_MODE9:
1109		case mmGB_MACROTILE_MODE10:
1110		case mmGB_MACROTILE_MODE11:
1111		case mmGB_MACROTILE_MODE12:
1112		case mmGB_MACROTILE_MODE13:
1113		case mmGB_MACROTILE_MODE14:
1114		case mmGB_MACROTILE_MODE15:
1115			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1116			return adev->gfx.config.macrotile_mode_array[idx];
1117		default:
1118			return RREG32(reg_offset);
1119		}
1120	}
1121}
1122
1123static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1124			     u32 sh_num, u32 reg_offset, u32 *value)
1125{
1126	uint32_t i;
1127
1128	*value = 0;
1129	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1130		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1131
1132		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1133			continue;
1134
1135		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1136						reg_offset);
 
 
 
1137		return 0;
1138	}
1139	return -EINVAL;
1140}
1141
1142struct kv_reset_save_regs {
1143	u32 gmcon_reng_execute;
1144	u32 gmcon_misc;
1145	u32 gmcon_misc3;
1146};
1147
1148static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1149				   struct kv_reset_save_regs *save)
1150{
1151	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1152	save->gmcon_misc = RREG32(mmGMCON_MISC);
1153	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1154
1155	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1156		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1157	WREG32(mmGMCON_MISC, save->gmcon_misc &
1158		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1159			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1160}
1161
1162static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1163				      struct kv_reset_save_regs *save)
1164{
1165	int i;
1166
1167	WREG32(mmGMCON_PGFSM_WRITE, 0);
1168	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1169
1170	for (i = 0; i < 5; i++)
1171		WREG32(mmGMCON_PGFSM_WRITE, 0);
1172
1173	WREG32(mmGMCON_PGFSM_WRITE, 0);
1174	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1175
1176	for (i = 0; i < 5; i++)
1177		WREG32(mmGMCON_PGFSM_WRITE, 0);
1178
1179	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1180	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1181
1182	for (i = 0; i < 5; i++)
1183		WREG32(mmGMCON_PGFSM_WRITE, 0);
1184
1185	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1186	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1187
1188	for (i = 0; i < 5; i++)
1189		WREG32(mmGMCON_PGFSM_WRITE, 0);
1190
1191	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1192	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1193
1194	for (i = 0; i < 5; i++)
1195		WREG32(mmGMCON_PGFSM_WRITE, 0);
1196
1197	WREG32(mmGMCON_PGFSM_WRITE, 0);
1198	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1199
1200	for (i = 0; i < 5; i++)
1201		WREG32(mmGMCON_PGFSM_WRITE, 0);
1202
1203	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1204	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1205
1206	for (i = 0; i < 5; i++)
1207		WREG32(mmGMCON_PGFSM_WRITE, 0);
1208
1209	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1210	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1211
1212	for (i = 0; i < 5; i++)
1213		WREG32(mmGMCON_PGFSM_WRITE, 0);
1214
1215	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1216	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1217
1218	for (i = 0; i < 5; i++)
1219		WREG32(mmGMCON_PGFSM_WRITE, 0);
1220
1221	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1222	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1223
1224	for (i = 0; i < 5; i++)
1225		WREG32(mmGMCON_PGFSM_WRITE, 0);
1226
1227	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1228	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1229
1230	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1231	WREG32(mmGMCON_MISC, save->gmcon_misc);
1232	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1233}
1234
1235static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1236{
1237	struct kv_reset_save_regs kv_save = { 0 };
1238	u32 i;
1239	int r = -EINVAL;
1240
1241	dev_info(adev->dev, "GPU pci config reset\n");
1242
1243	if (adev->flags & AMD_IS_APU)
1244		kv_save_regs_for_reset(adev, &kv_save);
1245
1246	/* disable BM */
1247	pci_clear_master(adev->pdev);
1248	/* reset */
1249	amdgpu_device_pci_config_reset(adev);
1250
1251	udelay(100);
1252
1253	/* wait for asic to come out of reset */
1254	for (i = 0; i < adev->usec_timeout; i++) {
1255		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1256			/* enable BM */
1257			pci_set_master(adev->pdev);
1258			adev->has_hw_reset = true;
1259			r = 0;
1260			break;
1261		}
1262		udelay(1);
1263	}
1264
1265	/* does asic init need to be run first??? */
1266	if (adev->flags & AMD_IS_APU)
1267		kv_restore_regs_for_reset(adev, &kv_save);
1268
1269	return r;
1270}
1271
1272/**
1273 * cik_asic_reset - soft reset GPU
1274 *
1275 * @adev: amdgpu_device pointer
1276 *
1277 * Look up which blocks are hung and attempt
1278 * to reset them.
1279 * Returns 0 for success.
1280 */
1281static int cik_asic_reset(struct amdgpu_device *adev)
1282{
1283	int r;
1284
1285	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1286
1287	r = cik_gpu_pci_config_reset(adev);
1288
1289	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1290
1291	return r;
1292}
1293
1294static enum amd_reset_method
1295cik_asic_reset_method(struct amdgpu_device *adev)
1296{
1297	return AMD_RESET_METHOD_LEGACY;
1298}
1299
1300static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1301{
1302	return RREG32(mmCONFIG_MEMSIZE);
1303}
1304
1305static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1306			      u32 cntl_reg, u32 status_reg)
1307{
1308	int r, i;
1309	struct atom_clock_dividers dividers;
1310	uint32_t tmp;
1311
1312	r = amdgpu_atombios_get_clock_dividers(adev,
1313					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1314					       clock, false, &dividers);
1315	if (r)
1316		return r;
1317
1318	tmp = RREG32_SMC(cntl_reg);
1319	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1320		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1321	tmp |= dividers.post_divider;
1322	WREG32_SMC(cntl_reg, tmp);
1323
1324	for (i = 0; i < 100; i++) {
1325		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1326			break;
1327		mdelay(10);
1328	}
1329	if (i == 100)
1330		return -ETIMEDOUT;
1331
1332	return 0;
1333}
1334
1335static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1336{
1337	int r = 0;
1338
1339	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1340	if (r)
1341		return r;
1342
1343	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1344	return r;
1345}
1346
1347static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1348{
1349	int r, i;
1350	struct atom_clock_dividers dividers;
1351	u32 tmp;
1352
1353	r = amdgpu_atombios_get_clock_dividers(adev,
1354					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1355					       ecclk, false, &dividers);
1356	if (r)
1357		return r;
1358
1359	for (i = 0; i < 100; i++) {
1360		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1361			break;
1362		mdelay(10);
1363	}
1364	if (i == 100)
1365		return -ETIMEDOUT;
1366
1367	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1368	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1369		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1370	tmp |= dividers.post_divider;
1371	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1372
1373	for (i = 0; i < 100; i++) {
1374		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1375			break;
1376		mdelay(10);
1377	}
1378	if (i == 100)
1379		return -ETIMEDOUT;
1380
1381	return 0;
1382}
1383
1384static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1385{
1386	struct pci_dev *root = adev->pdev->bus->self;
1387	int bridge_pos, gpu_pos;
1388	u32 speed_cntl, current_data_rate;
1389	int i;
1390	u16 tmp16;
1391
1392	if (pci_is_root_bus(adev->pdev->bus))
1393		return;
1394
1395	if (amdgpu_pcie_gen2 == 0)
1396		return;
1397
1398	if (adev->flags & AMD_IS_APU)
1399		return;
1400
1401	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1402					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1403		return;
1404
1405	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1406	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1407		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1408	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1409		if (current_data_rate == 2) {
1410			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1411			return;
1412		}
1413		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1414	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1415		if (current_data_rate == 1) {
1416			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1417			return;
1418		}
1419		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1420	}
1421
1422	bridge_pos = pci_pcie_cap(root);
1423	if (!bridge_pos)
1424		return;
1425
1426	gpu_pos = pci_pcie_cap(adev->pdev);
1427	if (!gpu_pos)
1428		return;
1429
1430	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1431		/* re-try equalization if gen3 is not already enabled */
1432		if (current_data_rate != 2) {
1433			u16 bridge_cfg, gpu_cfg;
1434			u16 bridge_cfg2, gpu_cfg2;
1435			u32 max_lw, current_lw, tmp;
1436
1437			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1438			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1439
1440			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1441			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1442
1443			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1444			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1445
1446			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1447			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1448				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1449			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1450				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1451
1452			if (current_lw < max_lw) {
1453				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1454				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1455					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1456						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1457					tmp |= (max_lw <<
1458						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1459					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1460					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1461					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1462					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1463				}
1464			}
1465
1466			for (i = 0; i < 10; i++) {
1467				/* check status */
1468				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1469				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1470					break;
1471
1472				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1473				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1474
1475				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1476				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1477
1478				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1479				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1480				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1481
1482				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1483				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1484				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1485
1486				msleep(100);
1487
1488				/* linkctl */
1489				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1490				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1491				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1492				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1493
1494				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1495				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1496				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1497				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1498
1499				/* linkctl2 */
1500				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1501				tmp16 &= ~((1 << 4) | (7 << 9));
1502				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1503				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1504
1505				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1506				tmp16 &= ~((1 << 4) | (7 << 9));
1507				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1508				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1509
1510				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1511				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1512				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1513			}
1514		}
1515	}
1516
1517	/* set the link speed */
1518	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1519		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1520	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1521	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1522
1523	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1524	tmp16 &= ~0xf;
1525	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1526		tmp16 |= 3; /* gen3 */
1527	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1528		tmp16 |= 2; /* gen2 */
1529	else
1530		tmp16 |= 1; /* gen1 */
1531	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1532
1533	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1534	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1535	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1536
1537	for (i = 0; i < adev->usec_timeout; i++) {
1538		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1539		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1540			break;
1541		udelay(1);
1542	}
1543}
1544
1545static void cik_program_aspm(struct amdgpu_device *adev)
1546{
1547	u32 data, orig;
1548	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1549	bool disable_clkreq = false;
1550
1551	if (amdgpu_aspm == 0)
1552		return;
1553
1554	if (pci_is_root_bus(adev->pdev->bus))
1555		return;
1556
1557	/* XXX double check APUs */
1558	if (adev->flags & AMD_IS_APU)
1559		return;
1560
1561	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1562	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1563	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1564		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1565	if (orig != data)
1566		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1567
1568	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1569	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1570	if (orig != data)
1571		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1572
1573	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1574	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1575	if (orig != data)
1576		WREG32_PCIE(ixPCIE_P_CNTL, data);
1577
1578	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1579	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1580		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1581	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1582	if (!disable_l0s)
1583		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1584
1585	if (!disable_l1) {
1586		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1587		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1588		if (orig != data)
1589			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1590
1591		if (!disable_plloff_in_l1) {
1592			bool clk_req_support;
1593
1594			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1595			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1596				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1597			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1598				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1599			if (orig != data)
1600				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1601
1602			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1603			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1604				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1605			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1606				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1607			if (orig != data)
1608				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1609
1610			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1611			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1612				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1613			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1614				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1615			if (orig != data)
1616				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1617
1618			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1619			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1620				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1621			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1622				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1623			if (orig != data)
1624				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1625
1626			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1627			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1628			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1629			if (orig != data)
1630				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1631
1632			if (!disable_clkreq) {
1633				struct pci_dev *root = adev->pdev->bus->self;
1634				u32 lnkcap;
1635
1636				clk_req_support = false;
1637				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1638				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1639					clk_req_support = true;
1640			} else {
1641				clk_req_support = false;
1642			}
1643
1644			if (clk_req_support) {
1645				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1646				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1647					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1648				if (orig != data)
1649					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1650
1651				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1652				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1653					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1654				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1655					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1656				if (orig != data)
1657					WREG32_SMC(ixTHM_CLK_CNTL, data);
1658
1659				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1660				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1661					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1662				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1663					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1664				if (orig != data)
1665					WREG32_SMC(ixMISC_CLK_CTRL, data);
1666
1667				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1668				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1669				if (orig != data)
1670					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1671
1672				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1673				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1674				if (orig != data)
1675					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1676
1677				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1678				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1679				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1680				if (orig != data)
1681					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1682			}
1683		}
1684	} else {
1685		if (orig != data)
1686			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1687	}
1688
1689	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1690	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1691		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1692		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1693	if (orig != data)
1694		WREG32_PCIE(ixPCIE_CNTL2, data);
1695
1696	if (!disable_l0s) {
1697		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1698		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1699				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1700			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1701			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1702			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1703				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1704				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1705				if (orig != data)
1706					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1707			}
1708		}
1709	}
1710}
1711
1712static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1713{
1714	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1715		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1716}
1717
1718static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1719{
1720	if (is_virtual_machine()) /* passthrough mode */
1721		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1722}
1723
1724static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1725{
1726	if (!ring || !ring->funcs->emit_wreg) {
1727		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1728		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1729	} else {
1730		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1731	}
1732}
1733
1734static void cik_invalidate_hdp(struct amdgpu_device *adev,
1735			       struct amdgpu_ring *ring)
1736{
1737	if (!ring || !ring->funcs->emit_wreg) {
1738		WREG32(mmHDP_DEBUG0, 1);
1739		RREG32(mmHDP_DEBUG0);
1740	} else {
1741		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1742	}
1743}
1744
1745static bool cik_need_full_reset(struct amdgpu_device *adev)
1746{
1747	/* change this when we support soft reset */
1748	return true;
1749}
1750
1751static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1752			       uint64_t *count1)
1753{
1754	uint32_t perfctr = 0;
1755	uint64_t cnt0_of, cnt1_of;
1756	int tmp;
1757
1758	/* This reports 0 on APUs, so return to avoid writing/reading registers
1759	 * that may or may not be different from their GPU counterparts
1760	 */
1761	if (adev->flags & AMD_IS_APU)
1762		return;
1763
1764	/* Set the 2 events that we wish to watch, defined above */
1765	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1766	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1767	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1768
1769	/* Write to enable desired perf counters */
1770	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1771	/* Zero out and enable the perf counters
1772	 * Write 0x5:
1773	 * Bit 0 = Start all counters(1)
1774	 * Bit 2 = Global counter reset enable(1)
1775	 */
1776	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1777
1778	msleep(1000);
1779
1780	/* Load the shadow and disable the perf counters
1781	 * Write 0x2:
1782	 * Bit 0 = Stop counters(0)
1783	 * Bit 1 = Load the shadow counters(1)
1784	 */
1785	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1786
1787	/* Read register values to get any >32bit overflow */
1788	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1789	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1790	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1791
1792	/* Get the values and add the overflow */
1793	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1794	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1795}
1796
1797static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1798{
1799	u32 clock_cntl, pc;
1800
1801	if (adev->flags & AMD_IS_APU)
1802		return false;
1803
1804	/* check if the SMC is already running */
1805	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1806	pc = RREG32_SMC(ixSMC_PC_C);
1807	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1808	    (0x20100 <= pc))
1809		return true;
1810
1811	return false;
1812}
1813
1814static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1815{
1816	uint64_t nak_r, nak_g;
1817
1818	/* Get the number of NAKs received and generated */
1819	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1820	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1821
1822	/* Add the total number of NAKs, i.e the number of replays */
1823	return (nak_r + nak_g);
1824}
1825
1826static const struct amdgpu_asic_funcs cik_asic_funcs =
1827{
1828	.read_disabled_bios = &cik_read_disabled_bios,
1829	.read_bios_from_rom = &cik_read_bios_from_rom,
 
1830	.read_register = &cik_read_register,
1831	.reset = &cik_asic_reset,
1832	.reset_method = &cik_asic_reset_method,
1833	.set_vga_state = &cik_vga_set_state,
1834	.get_xclk = &cik_get_xclk,
1835	.set_uvd_clocks = &cik_set_uvd_clocks,
1836	.set_vce_clocks = &cik_set_vce_clocks,
1837	.get_config_memsize = &cik_get_config_memsize,
1838	.flush_hdp = &cik_flush_hdp,
1839	.invalidate_hdp = &cik_invalidate_hdp,
1840	.need_full_reset = &cik_need_full_reset,
1841	.init_doorbell_index = &legacy_doorbell_index_init,
1842	.get_pcie_usage = &cik_get_pcie_usage,
1843	.need_reset_on_init = &cik_need_reset_on_init,
1844	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1845};
1846
1847static int cik_common_early_init(void *handle)
1848{
1849	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1850
1851	adev->smc_rreg = &cik_smc_rreg;
1852	adev->smc_wreg = &cik_smc_wreg;
1853	adev->pcie_rreg = &cik_pcie_rreg;
1854	adev->pcie_wreg = &cik_pcie_wreg;
1855	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1856	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1857	adev->didt_rreg = &cik_didt_rreg;
1858	adev->didt_wreg = &cik_didt_wreg;
1859
1860	adev->asic_funcs = &cik_asic_funcs;
1861
1862	adev->rev_id = cik_get_rev_id(adev);
1863	adev->external_rev_id = 0xFF;
1864	switch (adev->asic_type) {
1865	case CHIP_BONAIRE:
1866		adev->cg_flags =
1867			AMD_CG_SUPPORT_GFX_MGCG |
1868			AMD_CG_SUPPORT_GFX_MGLS |
1869			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1870			AMD_CG_SUPPORT_GFX_CGLS |
1871			AMD_CG_SUPPORT_GFX_CGTS |
1872			AMD_CG_SUPPORT_GFX_CGTS_LS |
1873			AMD_CG_SUPPORT_GFX_CP_LS |
1874			AMD_CG_SUPPORT_MC_LS |
1875			AMD_CG_SUPPORT_MC_MGCG |
1876			AMD_CG_SUPPORT_SDMA_MGCG |
1877			AMD_CG_SUPPORT_SDMA_LS |
1878			AMD_CG_SUPPORT_BIF_LS |
1879			AMD_CG_SUPPORT_VCE_MGCG |
1880			AMD_CG_SUPPORT_UVD_MGCG |
1881			AMD_CG_SUPPORT_HDP_LS |
1882			AMD_CG_SUPPORT_HDP_MGCG;
1883		adev->pg_flags = 0;
1884		adev->external_rev_id = adev->rev_id + 0x14;
1885		break;
1886	case CHIP_HAWAII:
1887		adev->cg_flags =
1888			AMD_CG_SUPPORT_GFX_MGCG |
1889			AMD_CG_SUPPORT_GFX_MGLS |
1890			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1891			AMD_CG_SUPPORT_GFX_CGLS |
1892			AMD_CG_SUPPORT_GFX_CGTS |
1893			AMD_CG_SUPPORT_GFX_CP_LS |
1894			AMD_CG_SUPPORT_MC_LS |
1895			AMD_CG_SUPPORT_MC_MGCG |
1896			AMD_CG_SUPPORT_SDMA_MGCG |
1897			AMD_CG_SUPPORT_SDMA_LS |
1898			AMD_CG_SUPPORT_BIF_LS |
1899			AMD_CG_SUPPORT_VCE_MGCG |
1900			AMD_CG_SUPPORT_UVD_MGCG |
1901			AMD_CG_SUPPORT_HDP_LS |
1902			AMD_CG_SUPPORT_HDP_MGCG;
1903		adev->pg_flags = 0;
1904		adev->external_rev_id = 0x28;
1905		break;
1906	case CHIP_KAVERI:
1907		adev->cg_flags =
1908			AMD_CG_SUPPORT_GFX_MGCG |
1909			AMD_CG_SUPPORT_GFX_MGLS |
1910			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1911			AMD_CG_SUPPORT_GFX_CGLS |
1912			AMD_CG_SUPPORT_GFX_CGTS |
1913			AMD_CG_SUPPORT_GFX_CGTS_LS |
1914			AMD_CG_SUPPORT_GFX_CP_LS |
1915			AMD_CG_SUPPORT_SDMA_MGCG |
1916			AMD_CG_SUPPORT_SDMA_LS |
1917			AMD_CG_SUPPORT_BIF_LS |
1918			AMD_CG_SUPPORT_VCE_MGCG |
1919			AMD_CG_SUPPORT_UVD_MGCG |
1920			AMD_CG_SUPPORT_HDP_LS |
1921			AMD_CG_SUPPORT_HDP_MGCG;
1922		adev->pg_flags =
1923			/*AMD_PG_SUPPORT_GFX_PG |
1924			  AMD_PG_SUPPORT_GFX_SMG |
1925			  AMD_PG_SUPPORT_GFX_DMG |*/
1926			AMD_PG_SUPPORT_UVD |
1927			AMD_PG_SUPPORT_VCE |
1928			/*  AMD_PG_SUPPORT_CP |
1929			  AMD_PG_SUPPORT_GDS |
1930			  AMD_PG_SUPPORT_RLC_SMU_HS |
1931			  AMD_PG_SUPPORT_ACP |
1932			  AMD_PG_SUPPORT_SAMU |*/
1933			0;
1934		if (adev->pdev->device == 0x1312 ||
1935			adev->pdev->device == 0x1316 ||
1936			adev->pdev->device == 0x1317)
1937			adev->external_rev_id = 0x41;
1938		else
1939			adev->external_rev_id = 0x1;
1940		break;
1941	case CHIP_KABINI:
1942	case CHIP_MULLINS:
1943		adev->cg_flags =
1944			AMD_CG_SUPPORT_GFX_MGCG |
1945			AMD_CG_SUPPORT_GFX_MGLS |
1946			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1947			AMD_CG_SUPPORT_GFX_CGLS |
1948			AMD_CG_SUPPORT_GFX_CGTS |
1949			AMD_CG_SUPPORT_GFX_CGTS_LS |
1950			AMD_CG_SUPPORT_GFX_CP_LS |
1951			AMD_CG_SUPPORT_SDMA_MGCG |
1952			AMD_CG_SUPPORT_SDMA_LS |
1953			AMD_CG_SUPPORT_BIF_LS |
1954			AMD_CG_SUPPORT_VCE_MGCG |
1955			AMD_CG_SUPPORT_UVD_MGCG |
1956			AMD_CG_SUPPORT_HDP_LS |
1957			AMD_CG_SUPPORT_HDP_MGCG;
1958		adev->pg_flags =
1959			/*AMD_PG_SUPPORT_GFX_PG |
1960			  AMD_PG_SUPPORT_GFX_SMG | */
1961			AMD_PG_SUPPORT_UVD |
1962			/*AMD_PG_SUPPORT_VCE |
1963			  AMD_PG_SUPPORT_CP |
1964			  AMD_PG_SUPPORT_GDS |
1965			  AMD_PG_SUPPORT_RLC_SMU_HS |
1966			  AMD_PG_SUPPORT_SAMU |*/
1967			0;
1968		if (adev->asic_type == CHIP_KABINI) {
1969			if (adev->rev_id == 0)
1970				adev->external_rev_id = 0x81;
1971			else if (adev->rev_id == 1)
1972				adev->external_rev_id = 0x82;
1973			else if (adev->rev_id == 2)
1974				adev->external_rev_id = 0x85;
1975		} else
1976			adev->external_rev_id = adev->rev_id + 0xa1;
1977		break;
1978	default:
1979		/* FIXME: not supported yet */
1980		return -EINVAL;
1981	}
1982
 
 
1983	return 0;
1984}
1985
1986static int cik_common_sw_init(void *handle)
1987{
1988	return 0;
1989}
1990
1991static int cik_common_sw_fini(void *handle)
1992{
1993	return 0;
1994}
1995
1996static int cik_common_hw_init(void *handle)
1997{
1998	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1999
2000	/* move the golden regs per IP block */
2001	cik_init_golden_registers(adev);
2002	/* enable pcie gen2/3 link */
2003	cik_pcie_gen3_enable(adev);
2004	/* enable aspm */
2005	cik_program_aspm(adev);
2006
2007	return 0;
2008}
2009
2010static int cik_common_hw_fini(void *handle)
2011{
2012	return 0;
2013}
2014
2015static int cik_common_suspend(void *handle)
2016{
2017	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2018
 
 
2019	return cik_common_hw_fini(adev);
2020}
2021
2022static int cik_common_resume(void *handle)
2023{
 
2024	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2025
2026	return cik_common_hw_init(adev);
 
 
 
 
2027}
2028
2029static bool cik_common_is_idle(void *handle)
2030{
2031	return true;
2032}
2033
2034static int cik_common_wait_for_idle(void *handle)
2035{
2036	return 0;
2037}
2038
2039static int cik_common_soft_reset(void *handle)
2040{
2041	/* XXX hard reset?? */
2042	return 0;
2043}
2044
2045static int cik_common_set_clockgating_state(void *handle,
2046					    enum amd_clockgating_state state)
2047{
2048	return 0;
2049}
2050
2051static int cik_common_set_powergating_state(void *handle,
2052					    enum amd_powergating_state state)
2053{
2054	return 0;
2055}
2056
2057static const struct amd_ip_funcs cik_common_ip_funcs = {
2058	.name = "cik_common",
2059	.early_init = cik_common_early_init,
2060	.late_init = NULL,
2061	.sw_init = cik_common_sw_init,
2062	.sw_fini = cik_common_sw_fini,
2063	.hw_init = cik_common_hw_init,
2064	.hw_fini = cik_common_hw_fini,
2065	.suspend = cik_common_suspend,
2066	.resume = cik_common_resume,
2067	.is_idle = cik_common_is_idle,
2068	.wait_for_idle = cik_common_wait_for_idle,
2069	.soft_reset = cik_common_soft_reset,
2070	.set_clockgating_state = cik_common_set_clockgating_state,
2071	.set_powergating_state = cik_common_set_powergating_state,
2072};
2073
2074static const struct amdgpu_ip_block_version cik_common_ip_block =
2075{
2076	.type = AMD_IP_BLOCK_TYPE_COMMON,
2077	.major = 1,
2078	.minor = 0,
2079	.rev = 0,
2080	.funcs = &cik_common_ip_funcs,
2081};
2082
2083int cik_set_ip_blocks(struct amdgpu_device *adev)
2084{
2085	cik_detect_hw_virtualization(adev);
2086
2087	switch (adev->asic_type) {
2088	case CHIP_BONAIRE:
2089		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2090		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2091		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2092		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2093		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2094		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2095		if (adev->enable_virtual_display)
2096			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2097#if defined(CONFIG_DRM_AMD_DC)
2098		else if (amdgpu_device_has_dc_support(adev))
2099			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2100#endif
2101		else
2102			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2103		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2104		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2105		break;
2106	case CHIP_HAWAII:
2107		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2108		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2109		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2110		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2111		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2112		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2113		if (adev->enable_virtual_display)
2114			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2115#if defined(CONFIG_DRM_AMD_DC)
2116		else if (amdgpu_device_has_dc_support(adev))
2117			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2118#endif
2119		else
2120			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2121		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2122		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2123		break;
2124	case CHIP_KAVERI:
2125		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2126		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2127		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2128		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2129		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2130		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2131		if (adev->enable_virtual_display)
2132			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2133#if defined(CONFIG_DRM_AMD_DC)
2134		else if (amdgpu_device_has_dc_support(adev))
2135			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2136#endif
2137		else
2138			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2139
2140		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2141		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
2142		break;
2143	case CHIP_KABINI:
2144	case CHIP_MULLINS:
2145		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2146		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2147		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2148		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2149		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2150		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2151		if (adev->enable_virtual_display)
2152			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2153#if defined(CONFIG_DRM_AMD_DC)
2154		else if (amdgpu_device_has_dc_support(adev))
2155			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2156#endif
2157		else
2158			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2159		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2160		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2161		break;
2162	default:
2163		/* FIXME: not supported yet */
2164		return -EINVAL;
2165	}
2166	return 0;
2167}