1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include <drm/amdgpu_drm.h>
  30
  31#include "amdgpu.h"
  32#include "amdgpu_atombios.h"
  33#include "amdgpu_ih.h"
  34#include "amdgpu_uvd.h"
  35#include "amdgpu_vce.h"
  36#include "cikd.h"
  37#include "atom.h"
  38#include "amd_pcie.h"
  39
  40#include "cik.h"
  41#include "gmc_v7_0.h"
  42#include "cik_ih.h"
  43#include "dce_v8_0.h"
  44#include "gfx_v7_0.h"
  45#include "cik_sdma.h"
  46#include "uvd_v4_2.h"
  47#include "vce_v2_0.h"
  48#include "cik_dpm.h"
  49
  50#include "uvd/uvd_4_2_d.h"
  51
  52#include "smu/smu_7_0_1_d.h"
  53#include "smu/smu_7_0_1_sh_mask.h"
  54
  55#include "dce/dce_8_0_d.h"
  56#include "dce/dce_8_0_sh_mask.h"
  57
  58#include "bif/bif_4_1_d.h"
  59#include "bif/bif_4_1_sh_mask.h"
  60
  61#include "gca/gfx_7_2_d.h"
  62#include "gca/gfx_7_2_enum.h"
  63#include "gca/gfx_7_2_sh_mask.h"
  64
  65#include "gmc/gmc_7_1_d.h"
  66#include "gmc/gmc_7_1_sh_mask.h"
  67
  68#include "oss/oss_2_0_d.h"
  69#include "oss/oss_2_0_sh_mask.h"
  70
  71#include "amdgpu_dm.h"
  72#include "amdgpu_amdkfd.h"
  73#include "amdgpu_vkms.h"
  74
  75static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
  76{
  77	{
  78		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
  79		.max_width = 2048,
  80		.max_height = 1152,
  81		.max_pixels_per_frame = 2048 * 1152,
  82		.max_level = 0,
  83	},
  84};
  85
  86static const struct amdgpu_video_codecs cik_video_codecs_encode =
  87{
  88	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
  89	.codec_array = cik_video_codecs_encode_array,
  90};
  91
  92static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
  93{
  94	{
  95		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
  96		.max_width = 2048,
  97		.max_height = 1152,
  98		.max_pixels_per_frame = 2048 * 1152,
  99		.max_level = 3,
 100	},
 101	{
 102		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 103		.max_width = 2048,
 104		.max_height = 1152,
 105		.max_pixels_per_frame = 2048 * 1152,
 106		.max_level = 5,
 107	},
 108	{
 109		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 110		.max_width = 2048,
 111		.max_height = 1152,
 112		.max_pixels_per_frame = 2048 * 1152,
 113		.max_level = 41,
 114	},
 115	{
 116		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 117		.max_width = 2048,
 118		.max_height = 1152,
 119		.max_pixels_per_frame = 2048 * 1152,
 120		.max_level = 4,
 121	},
 122};
 123
 124static const struct amdgpu_video_codecs cik_video_codecs_decode =
 125{
 126	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
 127	.codec_array = cik_video_codecs_decode_array,
 128};
 129
 130static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
 131				  const struct amdgpu_video_codecs **codecs)
 132{
 133	switch (adev->asic_type) {
 134	case CHIP_BONAIRE:
 135	case CHIP_HAWAII:
 136	case CHIP_KAVERI:
 137	case CHIP_KABINI:
 138	case CHIP_MULLINS:
 139		if (encode)
 140			*codecs = &cik_video_codecs_encode;
 141		else
 142			*codecs = &cik_video_codecs_decode;
 143		return 0;
 144	default:
 145		return -EINVAL;
 146	}
 147}
 148
 149/*
 150 * Indirect registers accessor
 151 */
 152static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 153{
 154	unsigned long flags;
 155	u32 r;
 156
 157	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 158	WREG32(mmPCIE_INDEX, reg);
 159	(void)RREG32(mmPCIE_INDEX);
 160	r = RREG32(mmPCIE_DATA);
 161	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162	return r;
 163}
 164
 165static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 166{
 167	unsigned long flags;
 168
 169	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 170	WREG32(mmPCIE_INDEX, reg);
 171	(void)RREG32(mmPCIE_INDEX);
 172	WREG32(mmPCIE_DATA, v);
 173	(void)RREG32(mmPCIE_DATA);
 174	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 175}
 176
 177static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 178{
 179	unsigned long flags;
 180	u32 r;
 181
 182	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 183	WREG32(mmSMC_IND_INDEX_0, (reg));
 184	r = RREG32(mmSMC_IND_DATA_0);
 185	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 186	return r;
 187}
 188
 189static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 190{
 191	unsigned long flags;
 192
 193	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 194	WREG32(mmSMC_IND_INDEX_0, (reg));
 195	WREG32(mmSMC_IND_DATA_0, (v));
 196	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 197}
 198
 199static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 200{
 201	unsigned long flags;
 202	u32 r;
 203
 204	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 205	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 206	r = RREG32(mmUVD_CTX_DATA);
 207	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 208	return r;
 209}
 210
 211static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 212{
 213	unsigned long flags;
 214
 215	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 216	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 217	WREG32(mmUVD_CTX_DATA, (v));
 218	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 219}
 220
 221static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 222{
 223	unsigned long flags;
 224	u32 r;
 225
 226	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 227	WREG32(mmDIDT_IND_INDEX, (reg));
 228	r = RREG32(mmDIDT_IND_DATA);
 229	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 230	return r;
 231}
 232
 233static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 234{
 235	unsigned long flags;
 236
 237	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 238	WREG32(mmDIDT_IND_INDEX, (reg));
 239	WREG32(mmDIDT_IND_DATA, (v));
 240	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 241}
 242
 243static const u32 bonaire_golden_spm_registers[] =
 244{
 245	0xc200, 0xe0ffffff, 0xe0000000
 246};
 247
 248static const u32 bonaire_golden_common_registers[] =
 249{
 250	0x31dc, 0xffffffff, 0x00000800,
 251	0x31dd, 0xffffffff, 0x00000800,
 252	0x31e6, 0xffffffff, 0x00007fbf,
 253	0x31e7, 0xffffffff, 0x00007faf
 254};
 255
 256static const u32 bonaire_golden_registers[] =
 257{
 258	0xcd5, 0x00000333, 0x00000333,
 259	0xcd4, 0x000c0fc0, 0x00040200,
 260	0x2684, 0x00010000, 0x00058208,
 261	0xf000, 0xffff1fff, 0x00140000,
 262	0xf080, 0xfdfc0fff, 0x00000100,
 263	0xf08d, 0x40000000, 0x40000200,
 264	0x260c, 0xffffffff, 0x00000000,
 265	0x260d, 0xf00fffff, 0x00000400,
 266	0x260e, 0x0002021c, 0x00020200,
 267	0x31e, 0x00000080, 0x00000000,
 268	0x16ec, 0x000000f0, 0x00000070,
 269	0x16f0, 0xf0311fff, 0x80300000,
 270	0x263e, 0x73773777, 0x12010001,
 271	0xd43, 0x00810000, 0x408af000,
 272	0x1c0c, 0x31000111, 0x00000011,
 273	0xbd2, 0x73773777, 0x12010001,
 274	0x883, 0x00007fb6, 0x0021a1b1,
 275	0x884, 0x00007fb6, 0x002021b1,
 276	0x860, 0x00007fb6, 0x00002191,
 277	0x886, 0x00007fb6, 0x002121b1,
 278	0x887, 0x00007fb6, 0x002021b1,
 279	0x877, 0x00007fb6, 0x00002191,
 280	0x878, 0x00007fb6, 0x00002191,
 281	0xd8a, 0x0000003f, 0x0000000a,
 282	0xd8b, 0x0000003f, 0x0000000a,
 283	0xab9, 0x00073ffe, 0x000022a2,
 284	0x903, 0x000007ff, 0x00000000,
 285	0x2285, 0xf000003f, 0x00000007,
 286	0x22fc, 0x00002001, 0x00000001,
 287	0x22c9, 0xffffffff, 0x00ffffff,
 288	0xc281, 0x0000ff0f, 0x00000000,
 289	0xa293, 0x07ffffff, 0x06000000,
 290	0x136, 0x00000fff, 0x00000100,
 291	0xf9e, 0x00000001, 0x00000002,
 292	0x2440, 0x03000000, 0x0362c688,
 293	0x2300, 0x000000ff, 0x00000001,
 294	0x390, 0x00001fff, 0x00001fff,
 295	0x2418, 0x0000007f, 0x00000020,
 296	0x2542, 0x00010000, 0x00010000,
 297	0x2b05, 0x000003ff, 0x000000f3,
 298	0x2b03, 0xffffffff, 0x00001032
 299};
 300
 301static const u32 bonaire_mgcg_cgcg_init[] =
 302{
 303	0x3108, 0xffffffff, 0xfffffffc,
 304	0xc200, 0xffffffff, 0xe0000000,
 305	0xf0a8, 0xffffffff, 0x00000100,
 306	0xf082, 0xffffffff, 0x00000100,
 307	0xf0b0, 0xffffffff, 0xc0000100,
 308	0xf0b2, 0xffffffff, 0xc0000100,
 309	0xf0b1, 0xffffffff, 0xc0000100,
 310	0x1579, 0xffffffff, 0x00600100,
 311	0xf0a0, 0xffffffff, 0x00000100,
 312	0xf085, 0xffffffff, 0x06000100,
 313	0xf088, 0xffffffff, 0x00000100,
 314	0xf086, 0xffffffff, 0x06000100,
 315	0xf081, 0xffffffff, 0x00000100,
 316	0xf0b8, 0xffffffff, 0x00000100,
 317	0xf089, 0xffffffff, 0x00000100,
 318	0xf080, 0xffffffff, 0x00000100,
 319	0xf08c, 0xffffffff, 0x00000100,
 320	0xf08d, 0xffffffff, 0x00000100,
 321	0xf094, 0xffffffff, 0x00000100,
 322	0xf095, 0xffffffff, 0x00000100,
 323	0xf096, 0xffffffff, 0x00000100,
 324	0xf097, 0xffffffff, 0x00000100,
 325	0xf098, 0xffffffff, 0x00000100,
 326	0xf09f, 0xffffffff, 0x00000100,
 327	0xf09e, 0xffffffff, 0x00000100,
 328	0xf084, 0xffffffff, 0x06000100,
 329	0xf0a4, 0xffffffff, 0x00000100,
 330	0xf09d, 0xffffffff, 0x00000100,
 331	0xf0ad, 0xffffffff, 0x00000100,
 332	0xf0ac, 0xffffffff, 0x00000100,
 333	0xf09c, 0xffffffff, 0x00000100,
 334	0xc200, 0xffffffff, 0xe0000000,
 335	0xf008, 0xffffffff, 0x00010000,
 336	0xf009, 0xffffffff, 0x00030002,
 337	0xf00a, 0xffffffff, 0x00040007,
 338	0xf00b, 0xffffffff, 0x00060005,
 339	0xf00c, 0xffffffff, 0x00090008,
 340	0xf00d, 0xffffffff, 0x00010000,
 341	0xf00e, 0xffffffff, 0x00030002,
 342	0xf00f, 0xffffffff, 0x00040007,
 343	0xf010, 0xffffffff, 0x00060005,
 344	0xf011, 0xffffffff, 0x00090008,
 345	0xf012, 0xffffffff, 0x00010000,
 346	0xf013, 0xffffffff, 0x00030002,
 347	0xf014, 0xffffffff, 0x00040007,
 348	0xf015, 0xffffffff, 0x00060005,
 349	0xf016, 0xffffffff, 0x00090008,
 350	0xf017, 0xffffffff, 0x00010000,
 351	0xf018, 0xffffffff, 0x00030002,
 352	0xf019, 0xffffffff, 0x00040007,
 353	0xf01a, 0xffffffff, 0x00060005,
 354	0xf01b, 0xffffffff, 0x00090008,
 355	0xf01c, 0xffffffff, 0x00010000,
 356	0xf01d, 0xffffffff, 0x00030002,
 357	0xf01e, 0xffffffff, 0x00040007,
 358	0xf01f, 0xffffffff, 0x00060005,
 359	0xf020, 0xffffffff, 0x00090008,
 360	0xf021, 0xffffffff, 0x00010000,
 361	0xf022, 0xffffffff, 0x00030002,
 362	0xf023, 0xffffffff, 0x00040007,
 363	0xf024, 0xffffffff, 0x00060005,
 364	0xf025, 0xffffffff, 0x00090008,
 365	0xf026, 0xffffffff, 0x00010000,
 366	0xf027, 0xffffffff, 0x00030002,
 367	0xf028, 0xffffffff, 0x00040007,
 368	0xf029, 0xffffffff, 0x00060005,
 369	0xf02a, 0xffffffff, 0x00090008,
 370	0xf000, 0xffffffff, 0x96e00200,
 371	0x21c2, 0xffffffff, 0x00900100,
 372	0x3109, 0xffffffff, 0x0020003f,
 373	0xe, 0xffffffff, 0x0140001c,
 374	0xf, 0x000f0000, 0x000f0000,
 375	0x88, 0xffffffff, 0xc060000c,
 376	0x89, 0xc0000fff, 0x00000100,
 377	0x3e4, 0xffffffff, 0x00000100,
 378	0x3e6, 0x00000101, 0x00000000,
 379	0x82a, 0xffffffff, 0x00000104,
 380	0x1579, 0xff000fff, 0x00000100,
 381	0xc33, 0xc0000fff, 0x00000104,
 382	0x3079, 0x00000001, 0x00000001,
 383	0x3403, 0xff000ff0, 0x00000100,
 384	0x3603, 0xff000ff0, 0x00000100
 385};
 386
 387static const u32 spectre_golden_spm_registers[] =
 388{
 389	0xc200, 0xe0ffffff, 0xe0000000
 390};
 391
 392static const u32 spectre_golden_common_registers[] =
 393{
 394	0x31dc, 0xffffffff, 0x00000800,
 395	0x31dd, 0xffffffff, 0x00000800,
 396	0x31e6, 0xffffffff, 0x00007fbf,
 397	0x31e7, 0xffffffff, 0x00007faf
 398};
 399
 400static const u32 spectre_golden_registers[] =
 401{
 402	0xf000, 0xffff1fff, 0x96940200,
 403	0xf003, 0xffff0001, 0xff000000,
 404	0xf080, 0xfffc0fff, 0x00000100,
 405	0x1bb6, 0x00010101, 0x00010000,
 406	0x260d, 0xf00fffff, 0x00000400,
 407	0x260e, 0xfffffffc, 0x00020200,
 408	0x16ec, 0x000000f0, 0x00000070,
 409	0x16f0, 0xf0311fff, 0x80300000,
 410	0x263e, 0x73773777, 0x12010001,
 411	0x26df, 0x00ff0000, 0x00fc0000,
 412	0xbd2, 0x73773777, 0x12010001,
 413	0x2285, 0xf000003f, 0x00000007,
 414	0x22c9, 0xffffffff, 0x00ffffff,
 415	0xa0d4, 0x3f3f3fff, 0x00000082,
 416	0xa0d5, 0x0000003f, 0x00000000,
 417	0xf9e, 0x00000001, 0x00000002,
 418	0x244f, 0xffff03df, 0x00000004,
 419	0x31da, 0x00000008, 0x00000008,
 420	0x2300, 0x000008ff, 0x00000800,
 421	0x2542, 0x00010000, 0x00010000,
 422	0x2b03, 0xffffffff, 0x54763210,
 423	0x853e, 0x01ff01ff, 0x00000002,
 424	0x8526, 0x007ff800, 0x00200000,
 425	0x8057, 0xffffffff, 0x00000f40,
 426	0xc24d, 0xffffffff, 0x00000001
 427};
 428
 429static const u32 spectre_mgcg_cgcg_init[] =
 430{
 431	0x3108, 0xffffffff, 0xfffffffc,
 432	0xc200, 0xffffffff, 0xe0000000,
 433	0xf0a8, 0xffffffff, 0x00000100,
 434	0xf082, 0xffffffff, 0x00000100,
 435	0xf0b0, 0xffffffff, 0x00000100,
 436	0xf0b2, 0xffffffff, 0x00000100,
 437	0xf0b1, 0xffffffff, 0x00000100,
 438	0x1579, 0xffffffff, 0x00600100,
 439	0xf0a0, 0xffffffff, 0x00000100,
 440	0xf085, 0xffffffff, 0x06000100,
 441	0xf088, 0xffffffff, 0x00000100,
 442	0xf086, 0xffffffff, 0x06000100,
 443	0xf081, 0xffffffff, 0x00000100,
 444	0xf0b8, 0xffffffff, 0x00000100,
 445	0xf089, 0xffffffff, 0x00000100,
 446	0xf080, 0xffffffff, 0x00000100,
 447	0xf08c, 0xffffffff, 0x00000100,
 448	0xf08d, 0xffffffff, 0x00000100,
 449	0xf094, 0xffffffff, 0x00000100,
 450	0xf095, 0xffffffff, 0x00000100,
 451	0xf096, 0xffffffff, 0x00000100,
 452	0xf097, 0xffffffff, 0x00000100,
 453	0xf098, 0xffffffff, 0x00000100,
 454	0xf09f, 0xffffffff, 0x00000100,
 455	0xf09e, 0xffffffff, 0x00000100,
 456	0xf084, 0xffffffff, 0x06000100,
 457	0xf0a4, 0xffffffff, 0x00000100,
 458	0xf09d, 0xffffffff, 0x00000100,
 459	0xf0ad, 0xffffffff, 0x00000100,
 460	0xf0ac, 0xffffffff, 0x00000100,
 461	0xf09c, 0xffffffff, 0x00000100,
 462	0xc200, 0xffffffff, 0xe0000000,
 463	0xf008, 0xffffffff, 0x00010000,
 464	0xf009, 0xffffffff, 0x00030002,
 465	0xf00a, 0xffffffff, 0x00040007,
 466	0xf00b, 0xffffffff, 0x00060005,
 467	0xf00c, 0xffffffff, 0x00090008,
 468	0xf00d, 0xffffffff, 0x00010000,
 469	0xf00e, 0xffffffff, 0x00030002,
 470	0xf00f, 0xffffffff, 0x00040007,
 471	0xf010, 0xffffffff, 0x00060005,
 472	0xf011, 0xffffffff, 0x00090008,
 473	0xf012, 0xffffffff, 0x00010000,
 474	0xf013, 0xffffffff, 0x00030002,
 475	0xf014, 0xffffffff, 0x00040007,
 476	0xf015, 0xffffffff, 0x00060005,
 477	0xf016, 0xffffffff, 0x00090008,
 478	0xf017, 0xffffffff, 0x00010000,
 479	0xf018, 0xffffffff, 0x00030002,
 480	0xf019, 0xffffffff, 0x00040007,
 481	0xf01a, 0xffffffff, 0x00060005,
 482	0xf01b, 0xffffffff, 0x00090008,
 483	0xf01c, 0xffffffff, 0x00010000,
 484	0xf01d, 0xffffffff, 0x00030002,
 485	0xf01e, 0xffffffff, 0x00040007,
 486	0xf01f, 0xffffffff, 0x00060005,
 487	0xf020, 0xffffffff, 0x00090008,
 488	0xf021, 0xffffffff, 0x00010000,
 489	0xf022, 0xffffffff, 0x00030002,
 490	0xf023, 0xffffffff, 0x00040007,
 491	0xf024, 0xffffffff, 0x00060005,
 492	0xf025, 0xffffffff, 0x00090008,
 493	0xf026, 0xffffffff, 0x00010000,
 494	0xf027, 0xffffffff, 0x00030002,
 495	0xf028, 0xffffffff, 0x00040007,
 496	0xf029, 0xffffffff, 0x00060005,
 497	0xf02a, 0xffffffff, 0x00090008,
 498	0xf02b, 0xffffffff, 0x00010000,
 499	0xf02c, 0xffffffff, 0x00030002,
 500	0xf02d, 0xffffffff, 0x00040007,
 501	0xf02e, 0xffffffff, 0x00060005,
 502	0xf02f, 0xffffffff, 0x00090008,
 503	0xf000, 0xffffffff, 0x96e00200,
 504	0x21c2, 0xffffffff, 0x00900100,
 505	0x3109, 0xffffffff, 0x0020003f,
 506	0xe, 0xffffffff, 0x0140001c,
 507	0xf, 0x000f0000, 0x000f0000,
 508	0x88, 0xffffffff, 0xc060000c,
 509	0x89, 0xc0000fff, 0x00000100,
 510	0x3e4, 0xffffffff, 0x00000100,
 511	0x3e6, 0x00000101, 0x00000000,
 512	0x82a, 0xffffffff, 0x00000104,
 513	0x1579, 0xff000fff, 0x00000100,
 514	0xc33, 0xc0000fff, 0x00000104,
 515	0x3079, 0x00000001, 0x00000001,
 516	0x3403, 0xff000ff0, 0x00000100,
 517	0x3603, 0xff000ff0, 0x00000100
 518};
 519
 520static const u32 kalindi_golden_spm_registers[] =
 521{
 522	0xc200, 0xe0ffffff, 0xe0000000
 523};
 524
 525static const u32 kalindi_golden_common_registers[] =
 526{
 527	0x31dc, 0xffffffff, 0x00000800,
 528	0x31dd, 0xffffffff, 0x00000800,
 529	0x31e6, 0xffffffff, 0x00007fbf,
 530	0x31e7, 0xffffffff, 0x00007faf
 531};
 532
 533static const u32 kalindi_golden_registers[] =
 534{
 535	0xf000, 0xffffdfff, 0x6e944040,
 536	0x1579, 0xff607fff, 0xfc000100,
 537	0xf088, 0xff000fff, 0x00000100,
 538	0xf089, 0xff000fff, 0x00000100,
 539	0xf080, 0xfffc0fff, 0x00000100,
 540	0x1bb6, 0x00010101, 0x00010000,
 541	0x260c, 0xffffffff, 0x00000000,
 542	0x260d, 0xf00fffff, 0x00000400,
 543	0x16ec, 0x000000f0, 0x00000070,
 544	0x16f0, 0xf0311fff, 0x80300000,
 545	0x263e, 0x73773777, 0x12010001,
 546	0x263f, 0xffffffff, 0x00000010,
 547	0x26df, 0x00ff0000, 0x00fc0000,
 548	0x200c, 0x00001f0f, 0x0000100a,
 549	0xbd2, 0x73773777, 0x12010001,
 550	0x902, 0x000fffff, 0x000c007f,
 551	0x2285, 0xf000003f, 0x00000007,
 552	0x22c9, 0x3fff3fff, 0x00ffcfff,
 553	0xc281, 0x0000ff0f, 0x00000000,
 554	0xa293, 0x07ffffff, 0x06000000,
 555	0x136, 0x00000fff, 0x00000100,
 556	0xf9e, 0x00000001, 0x00000002,
 557	0x31da, 0x00000008, 0x00000008,
 558	0x2300, 0x000000ff, 0x00000003,
 559	0x853e, 0x01ff01ff, 0x00000002,
 560	0x8526, 0x007ff800, 0x00200000,
 561	0x8057, 0xffffffff, 0x00000f40,
 562	0x2231, 0x001f3ae3, 0x00000082,
 563	0x2235, 0x0000001f, 0x00000010,
 564	0xc24d, 0xffffffff, 0x00000000
 565};
 566
 567static const u32 kalindi_mgcg_cgcg_init[] =
 568{
 569	0x3108, 0xffffffff, 0xfffffffc,
 570	0xc200, 0xffffffff, 0xe0000000,
 571	0xf0a8, 0xffffffff, 0x00000100,
 572	0xf082, 0xffffffff, 0x00000100,
 573	0xf0b0, 0xffffffff, 0x00000100,
 574	0xf0b2, 0xffffffff, 0x00000100,
 575	0xf0b1, 0xffffffff, 0x00000100,
 576	0x1579, 0xffffffff, 0x00600100,
 577	0xf0a0, 0xffffffff, 0x00000100,
 578	0xf085, 0xffffffff, 0x06000100,
 579	0xf088, 0xffffffff, 0x00000100,
 580	0xf086, 0xffffffff, 0x06000100,
 581	0xf081, 0xffffffff, 0x00000100,
 582	0xf0b8, 0xffffffff, 0x00000100,
 583	0xf089, 0xffffffff, 0x00000100,
 584	0xf080, 0xffffffff, 0x00000100,
 585	0xf08c, 0xffffffff, 0x00000100,
 586	0xf08d, 0xffffffff, 0x00000100,
 587	0xf094, 0xffffffff, 0x00000100,
 588	0xf095, 0xffffffff, 0x00000100,
 589	0xf096, 0xffffffff, 0x00000100,
 590	0xf097, 0xffffffff, 0x00000100,
 591	0xf098, 0xffffffff, 0x00000100,
 592	0xf09f, 0xffffffff, 0x00000100,
 593	0xf09e, 0xffffffff, 0x00000100,
 594	0xf084, 0xffffffff, 0x06000100,
 595	0xf0a4, 0xffffffff, 0x00000100,
 596	0xf09d, 0xffffffff, 0x00000100,
 597	0xf0ad, 0xffffffff, 0x00000100,
 598	0xf0ac, 0xffffffff, 0x00000100,
 599	0xf09c, 0xffffffff, 0x00000100,
 600	0xc200, 0xffffffff, 0xe0000000,
 601	0xf008, 0xffffffff, 0x00010000,
 602	0xf009, 0xffffffff, 0x00030002,
 603	0xf00a, 0xffffffff, 0x00040007,
 604	0xf00b, 0xffffffff, 0x00060005,
 605	0xf00c, 0xffffffff, 0x00090008,
 606	0xf00d, 0xffffffff, 0x00010000,
 607	0xf00e, 0xffffffff, 0x00030002,
 608	0xf00f, 0xffffffff, 0x00040007,
 609	0xf010, 0xffffffff, 0x00060005,
 610	0xf011, 0xffffffff, 0x00090008,
 611	0xf000, 0xffffffff, 0x96e00200,
 612	0x21c2, 0xffffffff, 0x00900100,
 613	0x3109, 0xffffffff, 0x0020003f,
 614	0xe, 0xffffffff, 0x0140001c,
 615	0xf, 0x000f0000, 0x000f0000,
 616	0x88, 0xffffffff, 0xc060000c,
 617	0x89, 0xc0000fff, 0x00000100,
 618	0x82a, 0xffffffff, 0x00000104,
 619	0x1579, 0xff000fff, 0x00000100,
 620	0xc33, 0xc0000fff, 0x00000104,
 621	0x3079, 0x00000001, 0x00000001,
 622	0x3403, 0xff000ff0, 0x00000100,
 623	0x3603, 0xff000ff0, 0x00000100
 624};
 625
 626static const u32 hawaii_golden_spm_registers[] =
 627{
 628	0xc200, 0xe0ffffff, 0xe0000000
 629};
 630
 631static const u32 hawaii_golden_common_registers[] =
 632{
 633	0xc200, 0xffffffff, 0xe0000000,
 634	0xa0d4, 0xffffffff, 0x3a00161a,
 635	0xa0d5, 0xffffffff, 0x0000002e,
 636	0x2684, 0xffffffff, 0x00018208,
 637	0x263e, 0xffffffff, 0x12011003
 638};
 639
 640static const u32 hawaii_golden_registers[] =
 641{
 642	0xcd5, 0x00000333, 0x00000333,
 643	0x2684, 0x00010000, 0x00058208,
 644	0x260c, 0xffffffff, 0x00000000,
 645	0x260d, 0xf00fffff, 0x00000400,
 646	0x260e, 0x0002021c, 0x00020200,
 647	0x31e, 0x00000080, 0x00000000,
 648	0x16ec, 0x000000f0, 0x00000070,
 649	0x16f0, 0xf0311fff, 0x80300000,
 650	0xd43, 0x00810000, 0x408af000,
 651	0x1c0c, 0x31000111, 0x00000011,
 652	0xbd2, 0x73773777, 0x12010001,
 653	0x848, 0x0000007f, 0x0000001b,
 654	0x877, 0x00007fb6, 0x00002191,
 655	0xd8a, 0x0000003f, 0x0000000a,
 656	0xd8b, 0x0000003f, 0x0000000a,
 657	0xab9, 0x00073ffe, 0x000022a2,
 658	0x903, 0x000007ff, 0x00000000,
 659	0x22fc, 0x00002001, 0x00000001,
 660	0x22c9, 0xffffffff, 0x00ffffff,
 661	0xc281, 0x0000ff0f, 0x00000000,
 662	0xa293, 0x07ffffff, 0x06000000,
 663	0xf9e, 0x00000001, 0x00000002,
 664	0x31da, 0x00000008, 0x00000008,
 665	0x31dc, 0x00000f00, 0x00000800,
 666	0x31dd, 0x00000f00, 0x00000800,
 667	0x31e6, 0x00ffffff, 0x00ff7fbf,
 668	0x31e7, 0x00ffffff, 0x00ff7faf,
 669	0x2300, 0x000000ff, 0x00000800,
 670	0x390, 0x00001fff, 0x00001fff,
 671	0x2418, 0x0000007f, 0x00000020,
 672	0x2542, 0x00010000, 0x00010000,
 673	0x2b80, 0x00100000, 0x000ff07c,
 674	0x2b05, 0x000003ff, 0x0000000f,
 675	0x2b04, 0xffffffff, 0x7564fdec,
 676	0x2b03, 0xffffffff, 0x3120b9a8,
 677	0x2b02, 0x20000000, 0x0f9c0000
 678};
 679
 680static const u32 hawaii_mgcg_cgcg_init[] =
 681{
 682	0x3108, 0xffffffff, 0xfffffffd,
 683	0xc200, 0xffffffff, 0xe0000000,
 684	0xf0a8, 0xffffffff, 0x00000100,
 685	0xf082, 0xffffffff, 0x00000100,
 686	0xf0b0, 0xffffffff, 0x00000100,
 687	0xf0b2, 0xffffffff, 0x00000100,
 688	0xf0b1, 0xffffffff, 0x00000100,
 689	0x1579, 0xffffffff, 0x00200100,
 690	0xf0a0, 0xffffffff, 0x00000100,
 691	0xf085, 0xffffffff, 0x06000100,
 692	0xf088, 0xffffffff, 0x00000100,
 693	0xf086, 0xffffffff, 0x06000100,
 694	0xf081, 0xffffffff, 0x00000100,
 695	0xf0b8, 0xffffffff, 0x00000100,
 696	0xf089, 0xffffffff, 0x00000100,
 697	0xf080, 0xffffffff, 0x00000100,
 698	0xf08c, 0xffffffff, 0x00000100,
 699	0xf08d, 0xffffffff, 0x00000100,
 700	0xf094, 0xffffffff, 0x00000100,
 701	0xf095, 0xffffffff, 0x00000100,
 702	0xf096, 0xffffffff, 0x00000100,
 703	0xf097, 0xffffffff, 0x00000100,
 704	0xf098, 0xffffffff, 0x00000100,
 705	0xf09f, 0xffffffff, 0x00000100,
 706	0xf09e, 0xffffffff, 0x00000100,
 707	0xf084, 0xffffffff, 0x06000100,
 708	0xf0a4, 0xffffffff, 0x00000100,
 709	0xf09d, 0xffffffff, 0x00000100,
 710	0xf0ad, 0xffffffff, 0x00000100,
 711	0xf0ac, 0xffffffff, 0x00000100,
 712	0xf09c, 0xffffffff, 0x00000100,
 713	0xc200, 0xffffffff, 0xe0000000,
 714	0xf008, 0xffffffff, 0x00010000,
 715	0xf009, 0xffffffff, 0x00030002,
 716	0xf00a, 0xffffffff, 0x00040007,
 717	0xf00b, 0xffffffff, 0x00060005,
 718	0xf00c, 0xffffffff, 0x00090008,
 719	0xf00d, 0xffffffff, 0x00010000,
 720	0xf00e, 0xffffffff, 0x00030002,
 721	0xf00f, 0xffffffff, 0x00040007,
 722	0xf010, 0xffffffff, 0x00060005,
 723	0xf011, 0xffffffff, 0x00090008,
 724	0xf012, 0xffffffff, 0x00010000,
 725	0xf013, 0xffffffff, 0x00030002,
 726	0xf014, 0xffffffff, 0x00040007,
 727	0xf015, 0xffffffff, 0x00060005,
 728	0xf016, 0xffffffff, 0x00090008,
 729	0xf017, 0xffffffff, 0x00010000,
 730	0xf018, 0xffffffff, 0x00030002,
 731	0xf019, 0xffffffff, 0x00040007,
 732	0xf01a, 0xffffffff, 0x00060005,
 733	0xf01b, 0xffffffff, 0x00090008,
 734	0xf01c, 0xffffffff, 0x00010000,
 735	0xf01d, 0xffffffff, 0x00030002,
 736	0xf01e, 0xffffffff, 0x00040007,
 737	0xf01f, 0xffffffff, 0x00060005,
 738	0xf020, 0xffffffff, 0x00090008,
 739	0xf021, 0xffffffff, 0x00010000,
 740	0xf022, 0xffffffff, 0x00030002,
 741	0xf023, 0xffffffff, 0x00040007,
 742	0xf024, 0xffffffff, 0x00060005,
 743	0xf025, 0xffffffff, 0x00090008,
 744	0xf026, 0xffffffff, 0x00010000,
 745	0xf027, 0xffffffff, 0x00030002,
 746	0xf028, 0xffffffff, 0x00040007,
 747	0xf029, 0xffffffff, 0x00060005,
 748	0xf02a, 0xffffffff, 0x00090008,
 749	0xf02b, 0xffffffff, 0x00010000,
 750	0xf02c, 0xffffffff, 0x00030002,
 751	0xf02d, 0xffffffff, 0x00040007,
 752	0xf02e, 0xffffffff, 0x00060005,
 753	0xf02f, 0xffffffff, 0x00090008,
 754	0xf030, 0xffffffff, 0x00010000,
 755	0xf031, 0xffffffff, 0x00030002,
 756	0xf032, 0xffffffff, 0x00040007,
 757	0xf033, 0xffffffff, 0x00060005,
 758	0xf034, 0xffffffff, 0x00090008,
 759	0xf035, 0xffffffff, 0x00010000,
 760	0xf036, 0xffffffff, 0x00030002,
 761	0xf037, 0xffffffff, 0x00040007,
 762	0xf038, 0xffffffff, 0x00060005,
 763	0xf039, 0xffffffff, 0x00090008,
 764	0xf03a, 0xffffffff, 0x00010000,
 765	0xf03b, 0xffffffff, 0x00030002,
 766	0xf03c, 0xffffffff, 0x00040007,
 767	0xf03d, 0xffffffff, 0x00060005,
 768	0xf03e, 0xffffffff, 0x00090008,
 769	0x30c6, 0xffffffff, 0x00020200,
 770	0xcd4, 0xffffffff, 0x00000200,
 771	0x570, 0xffffffff, 0x00000400,
 772	0x157a, 0xffffffff, 0x00000000,
 773	0xbd4, 0xffffffff, 0x00000902,
 774	0xf000, 0xffffffff, 0x96940200,
 775	0x21c2, 0xffffffff, 0x00900100,
 776	0x3109, 0xffffffff, 0x0020003f,
 777	0xe, 0xffffffff, 0x0140001c,
 778	0xf, 0x000f0000, 0x000f0000,
 779	0x88, 0xffffffff, 0xc060000c,
 780	0x89, 0xc0000fff, 0x00000100,
 781	0x3e4, 0xffffffff, 0x00000100,
 782	0x3e6, 0x00000101, 0x00000000,
 783	0x82a, 0xffffffff, 0x00000104,
 784	0x1579, 0xff000fff, 0x00000100,
 785	0xc33, 0xc0000fff, 0x00000104,
 786	0x3079, 0x00000001, 0x00000001,
 787	0x3403, 0xff000ff0, 0x00000100,
 788	0x3603, 0xff000ff0, 0x00000100
 789};
 790
 791static const u32 godavari_golden_registers[] =
 792{
 793	0x1579, 0xff607fff, 0xfc000100,
 794	0x1bb6, 0x00010101, 0x00010000,
 795	0x260c, 0xffffffff, 0x00000000,
 796	0x260c0, 0xf00fffff, 0x00000400,
 797	0x184c, 0xffffffff, 0x00010000,
 798	0x16ec, 0x000000f0, 0x00000070,
 799	0x16f0, 0xf0311fff, 0x80300000,
 800	0x263e, 0x73773777, 0x12010001,
 801	0x263f, 0xffffffff, 0x00000010,
 802	0x200c, 0x00001f0f, 0x0000100a,
 803	0xbd2, 0x73773777, 0x12010001,
 804	0x902, 0x000fffff, 0x000c007f,
 805	0x2285, 0xf000003f, 0x00000007,
 806	0x22c9, 0xffffffff, 0x00ff0fff,
 807	0xc281, 0x0000ff0f, 0x00000000,
 808	0xa293, 0x07ffffff, 0x06000000,
 809	0x136, 0x00000fff, 0x00000100,
 810	0x3405, 0x00010000, 0x00810001,
 811	0x3605, 0x00010000, 0x00810001,
 812	0xf9e, 0x00000001, 0x00000002,
 813	0x31da, 0x00000008, 0x00000008,
 814	0x31dc, 0x00000f00, 0x00000800,
 815	0x31dd, 0x00000f00, 0x00000800,
 816	0x31e6, 0x00ffffff, 0x00ff7fbf,
 817	0x31e7, 0x00ffffff, 0x00ff7faf,
 818	0x2300, 0x000000ff, 0x00000001,
 819	0x853e, 0x01ff01ff, 0x00000002,
 820	0x8526, 0x007ff800, 0x00200000,
 821	0x8057, 0xffffffff, 0x00000f40,
 822	0x2231, 0x001f3ae3, 0x00000082,
 823	0x2235, 0x0000001f, 0x00000010,
 824	0xc24d, 0xffffffff, 0x00000000
 825};
 826
 827static void cik_init_golden_registers(struct amdgpu_device *adev)
 828{
 829	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 830	mutex_lock(&adev->grbm_idx_mutex);
 831
 832	switch (adev->asic_type) {
 833	case CHIP_BONAIRE:
 834		amdgpu_device_program_register_sequence(adev,
 835							bonaire_mgcg_cgcg_init,
 836							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 837		amdgpu_device_program_register_sequence(adev,
 838							bonaire_golden_registers,
 839							ARRAY_SIZE(bonaire_golden_registers));
 840		amdgpu_device_program_register_sequence(adev,
 841							bonaire_golden_common_registers,
 842							ARRAY_SIZE(bonaire_golden_common_registers));
 843		amdgpu_device_program_register_sequence(adev,
 844							bonaire_golden_spm_registers,
 845							ARRAY_SIZE(bonaire_golden_spm_registers));
 846		break;
 847	case CHIP_KABINI:
 848		amdgpu_device_program_register_sequence(adev,
 849							kalindi_mgcg_cgcg_init,
 850							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 851		amdgpu_device_program_register_sequence(adev,
 852							kalindi_golden_registers,
 853							ARRAY_SIZE(kalindi_golden_registers));
 854		amdgpu_device_program_register_sequence(adev,
 855							kalindi_golden_common_registers,
 856							ARRAY_SIZE(kalindi_golden_common_registers));
 857		amdgpu_device_program_register_sequence(adev,
 858							kalindi_golden_spm_registers,
 859							ARRAY_SIZE(kalindi_golden_spm_registers));
 860		break;
 861	case CHIP_MULLINS:
 862		amdgpu_device_program_register_sequence(adev,
 863							kalindi_mgcg_cgcg_init,
 864							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 865		amdgpu_device_program_register_sequence(adev,
 866							godavari_golden_registers,
 867							ARRAY_SIZE(godavari_golden_registers));
 868		amdgpu_device_program_register_sequence(adev,
 869							kalindi_golden_common_registers,
 870							ARRAY_SIZE(kalindi_golden_common_registers));
 871		amdgpu_device_program_register_sequence(adev,
 872							kalindi_golden_spm_registers,
 873							ARRAY_SIZE(kalindi_golden_spm_registers));
 874		break;
 875	case CHIP_KAVERI:
 876		amdgpu_device_program_register_sequence(adev,
 877							spectre_mgcg_cgcg_init,
 878							ARRAY_SIZE(spectre_mgcg_cgcg_init));
 879		amdgpu_device_program_register_sequence(adev,
 880							spectre_golden_registers,
 881							ARRAY_SIZE(spectre_golden_registers));
 882		amdgpu_device_program_register_sequence(adev,
 883							spectre_golden_common_registers,
 884							ARRAY_SIZE(spectre_golden_common_registers));
 885		amdgpu_device_program_register_sequence(adev,
 886							spectre_golden_spm_registers,
 887							ARRAY_SIZE(spectre_golden_spm_registers));
 888		break;
 889	case CHIP_HAWAII:
 890		amdgpu_device_program_register_sequence(adev,
 891							hawaii_mgcg_cgcg_init,
 892							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 893		amdgpu_device_program_register_sequence(adev,
 894							hawaii_golden_registers,
 895							ARRAY_SIZE(hawaii_golden_registers));
 896		amdgpu_device_program_register_sequence(adev,
 897							hawaii_golden_common_registers,
 898							ARRAY_SIZE(hawaii_golden_common_registers));
 899		amdgpu_device_program_register_sequence(adev,
 900							hawaii_golden_spm_registers,
 901							ARRAY_SIZE(hawaii_golden_spm_registers));
 902		break;
 903	default:
 904		break;
 905	}
 906	mutex_unlock(&adev->grbm_idx_mutex);
 907}
 908
 909/**
 910 * cik_get_xclk - get the xclk
 911 *
 912 * @adev: amdgpu_device pointer
 913 *
 914 * Returns the reference clock used by the gfx engine
 915 * (CIK).
 916 */
 917static u32 cik_get_xclk(struct amdgpu_device *adev)
 918{
 919	u32 reference_clock = adev->clock.spll.reference_freq;
 920
 921	if (adev->flags & AMD_IS_APU) {
 922		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 923			return reference_clock / 2;
 924	} else {
 925		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 926			return reference_clock / 4;
 927	}
 928	return reference_clock;
 929}
 930
 931/**
 932 * cik_srbm_select - select specific register instances
 933 *
 934 * @adev: amdgpu_device pointer
 935 * @me: selected ME (micro engine)
 936 * @pipe: pipe
 937 * @queue: queue
 938 * @vmid: VMID
 939 *
 940 * Switches the currently active registers instances.  Some
 941 * registers are instanced per VMID, others are instanced per
 942 * me/pipe/queue combination.
 943 */
 944void cik_srbm_select(struct amdgpu_device *adev,
 945		     u32 me, u32 pipe, u32 queue, u32 vmid)
 946{
 947	u32 srbm_gfx_cntl =
 948		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 949		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 950		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 951		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 952	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 953}
 954
 955static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 956{
 957	uint32_t tmp;
 958
 959	tmp = RREG32(mmCONFIG_CNTL);
 960	if (!state)
 961		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 962	else
 963		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 964	WREG32(mmCONFIG_CNTL, tmp);
 965}
 966
 967static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 968{
 969	u32 bus_cntl;
 970	u32 d1vga_control = 0;
 971	u32 d2vga_control = 0;
 972	u32 vga_render_control = 0;
 973	u32 rom_cntl;
 974	bool r;
 975
 976	bus_cntl = RREG32(mmBUS_CNTL);
 977	if (adev->mode_info.num_crtc) {
 978		d1vga_control = RREG32(mmD1VGA_CONTROL);
 979		d2vga_control = RREG32(mmD2VGA_CONTROL);
 980		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 981	}
 982	rom_cntl = RREG32_SMC(ixROM_CNTL);
 983
 984	/* enable the rom */
 985	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 986	if (adev->mode_info.num_crtc) {
 987		/* Disable VGA mode */
 988		WREG32(mmD1VGA_CONTROL,
 989		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 990					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 991		WREG32(mmD2VGA_CONTROL,
 992		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 993					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 994		WREG32(mmVGA_RENDER_CONTROL,
 995		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 996	}
 997	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 998
 999	r = amdgpu_read_bios(adev);
1000
1001	/* restore regs */
1002	WREG32(mmBUS_CNTL, bus_cntl);
1003	if (adev->mode_info.num_crtc) {
1004		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007	}
1008	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009	return r;
1010}
1011
1012static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013				   u8 *bios, u32 length_bytes)
1014{
1015	u32 *dw_ptr;
1016	unsigned long flags;
1017	u32 i, length_dw;
1018
1019	if (bios == NULL)
1020		return false;
1021	if (length_bytes == 0)
1022		return false;
1023	/* APU vbios image is part of sbios image */
1024	if (adev->flags & AMD_IS_APU)
1025		return false;
1026
1027	dw_ptr = (u32 *)bios;
1028	length_dw = ALIGN(length_bytes, 4) / 4;
1029	/* take the smc lock since we are using the smc index */
1030	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031	/* set rom index to 0 */
1032	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033	WREG32(mmSMC_IND_DATA_0, 0);
1034	/* set index to data for continous read */
1035	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036	for (i = 0; i < length_dw; i++)
1037		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040	return true;
1041}
1042
1043static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044	{mmGRBM_STATUS},
1045	{mmGRBM_STATUS2},
1046	{mmGRBM_STATUS_SE0},
1047	{mmGRBM_STATUS_SE1},
1048	{mmGRBM_STATUS_SE2},
1049	{mmGRBM_STATUS_SE3},
1050	{mmSRBM_STATUS},
1051	{mmSRBM_STATUS2},
1052	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054	{mmCP_STAT},
1055	{mmCP_STALLED_STAT1},
1056	{mmCP_STALLED_STAT2},
1057	{mmCP_STALLED_STAT3},
1058	{mmCP_CPF_BUSY_STAT},
1059	{mmCP_CPF_STALLED_STAT1},
1060	{mmCP_CPF_STATUS},
1061	{mmCP_CPC_BUSY_STAT},
1062	{mmCP_CPC_STALLED_STAT1},
1063	{mmCP_CPC_STATUS},
1064	{mmGB_ADDR_CONFIG},
1065	{mmMC_ARB_RAMCFG},
1066	{mmGB_TILE_MODE0},
1067	{mmGB_TILE_MODE1},
1068	{mmGB_TILE_MODE2},
1069	{mmGB_TILE_MODE3},
1070	{mmGB_TILE_MODE4},
1071	{mmGB_TILE_MODE5},
1072	{mmGB_TILE_MODE6},
1073	{mmGB_TILE_MODE7},
1074	{mmGB_TILE_MODE8},
1075	{mmGB_TILE_MODE9},
1076	{mmGB_TILE_MODE10},
1077	{mmGB_TILE_MODE11},
1078	{mmGB_TILE_MODE12},
1079	{mmGB_TILE_MODE13},
1080	{mmGB_TILE_MODE14},
1081	{mmGB_TILE_MODE15},
1082	{mmGB_TILE_MODE16},
1083	{mmGB_TILE_MODE17},
1084	{mmGB_TILE_MODE18},
1085	{mmGB_TILE_MODE19},
1086	{mmGB_TILE_MODE20},
1087	{mmGB_TILE_MODE21},
1088	{mmGB_TILE_MODE22},
1089	{mmGB_TILE_MODE23},
1090	{mmGB_TILE_MODE24},
1091	{mmGB_TILE_MODE25},
1092	{mmGB_TILE_MODE26},
1093	{mmGB_TILE_MODE27},
1094	{mmGB_TILE_MODE28},
1095	{mmGB_TILE_MODE29},
1096	{mmGB_TILE_MODE30},
1097	{mmGB_TILE_MODE31},
1098	{mmGB_MACROTILE_MODE0},
1099	{mmGB_MACROTILE_MODE1},
1100	{mmGB_MACROTILE_MODE2},
1101	{mmGB_MACROTILE_MODE3},
1102	{mmGB_MACROTILE_MODE4},
1103	{mmGB_MACROTILE_MODE5},
1104	{mmGB_MACROTILE_MODE6},
1105	{mmGB_MACROTILE_MODE7},
1106	{mmGB_MACROTILE_MODE8},
1107	{mmGB_MACROTILE_MODE9},
1108	{mmGB_MACROTILE_MODE10},
1109	{mmGB_MACROTILE_MODE11},
1110	{mmGB_MACROTILE_MODE12},
1111	{mmGB_MACROTILE_MODE13},
1112	{mmGB_MACROTILE_MODE14},
1113	{mmGB_MACROTILE_MODE15},
1114	{mmCC_RB_BACKEND_DISABLE, true},
1115	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116	{mmGB_BACKEND_MAP, false},
1117	{mmPA_SC_RASTER_CONFIG, true},
1118	{mmPA_SC_RASTER_CONFIG_1, true},
1119};
1120
1121
1122static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123				       bool indexed, u32 se_num,
1124				       u32 sh_num, u32 reg_offset)
1125{
1126	if (indexed) {
1127		uint32_t val;
1128		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131		switch (reg_offset) {
1132		case mmCC_RB_BACKEND_DISABLE:
1133			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134		case mmGC_USER_RB_BACKEND_DISABLE:
1135			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136		case mmPA_SC_RASTER_CONFIG:
1137			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138		case mmPA_SC_RASTER_CONFIG_1:
1139			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140		}
1141
1142		mutex_lock(&adev->grbm_idx_mutex);
1143		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145
1146		val = RREG32(reg_offset);
1147
1148		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150		mutex_unlock(&adev->grbm_idx_mutex);
1151		return val;
1152	} else {
1153		unsigned idx;
1154
1155		switch (reg_offset) {
1156		case mmGB_ADDR_CONFIG:
1157			return adev->gfx.config.gb_addr_config;
1158		case mmMC_ARB_RAMCFG:
1159			return adev->gfx.config.mc_arb_ramcfg;
1160		case mmGB_TILE_MODE0:
1161		case mmGB_TILE_MODE1:
1162		case mmGB_TILE_MODE2:
1163		case mmGB_TILE_MODE3:
1164		case mmGB_TILE_MODE4:
1165		case mmGB_TILE_MODE5:
1166		case mmGB_TILE_MODE6:
1167		case mmGB_TILE_MODE7:
1168		case mmGB_TILE_MODE8:
1169		case mmGB_TILE_MODE9:
1170		case mmGB_TILE_MODE10:
1171		case mmGB_TILE_MODE11:
1172		case mmGB_TILE_MODE12:
1173		case mmGB_TILE_MODE13:
1174		case mmGB_TILE_MODE14:
1175		case mmGB_TILE_MODE15:
1176		case mmGB_TILE_MODE16:
1177		case mmGB_TILE_MODE17:
1178		case mmGB_TILE_MODE18:
1179		case mmGB_TILE_MODE19:
1180		case mmGB_TILE_MODE20:
1181		case mmGB_TILE_MODE21:
1182		case mmGB_TILE_MODE22:
1183		case mmGB_TILE_MODE23:
1184		case mmGB_TILE_MODE24:
1185		case mmGB_TILE_MODE25:
1186		case mmGB_TILE_MODE26:
1187		case mmGB_TILE_MODE27:
1188		case mmGB_TILE_MODE28:
1189		case mmGB_TILE_MODE29:
1190		case mmGB_TILE_MODE30:
1191		case mmGB_TILE_MODE31:
1192			idx = (reg_offset - mmGB_TILE_MODE0);
1193			return adev->gfx.config.tile_mode_array[idx];
1194		case mmGB_MACROTILE_MODE0:
1195		case mmGB_MACROTILE_MODE1:
1196		case mmGB_MACROTILE_MODE2:
1197		case mmGB_MACROTILE_MODE3:
1198		case mmGB_MACROTILE_MODE4:
1199		case mmGB_MACROTILE_MODE5:
1200		case mmGB_MACROTILE_MODE6:
1201		case mmGB_MACROTILE_MODE7:
1202		case mmGB_MACROTILE_MODE8:
1203		case mmGB_MACROTILE_MODE9:
1204		case mmGB_MACROTILE_MODE10:
1205		case mmGB_MACROTILE_MODE11:
1206		case mmGB_MACROTILE_MODE12:
1207		case mmGB_MACROTILE_MODE13:
1208		case mmGB_MACROTILE_MODE14:
1209		case mmGB_MACROTILE_MODE15:
1210			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211			return adev->gfx.config.macrotile_mode_array[idx];
1212		default:
1213			return RREG32(reg_offset);
1214		}
1215	}
1216}
1217
1218static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219			     u32 sh_num, u32 reg_offset, u32 *value)
1220{
1221	uint32_t i;
1222
1223	*value = 0;
1224	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228			continue;
1229
1230		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231						reg_offset);
1232		return 0;
1233	}
1234	return -EINVAL;
1235}
1236
1237struct kv_reset_save_regs {
1238	u32 gmcon_reng_execute;
1239	u32 gmcon_misc;
1240	u32 gmcon_misc3;
1241};
1242
1243static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244				   struct kv_reset_save_regs *save)
1245{
1246	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255}
1256
1257static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258				      struct kv_reset_save_regs *save)
1259{
1260	int i;
1261
1262	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265	for (i = 0; i < 5; i++)
1266		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271	for (i = 0; i < 5; i++)
1272		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277	for (i = 0; i < 5; i++)
1278		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283	for (i = 0; i < 5; i++)
1284		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289	for (i = 0; i < 5; i++)
1290		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295	for (i = 0; i < 5; i++)
1296		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301	for (i = 0; i < 5; i++)
1302		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307	for (i = 0; i < 5; i++)
1308		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313	for (i = 0; i < 5; i++)
1314		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319	for (i = 0; i < 5; i++)
1320		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328}
1329
1330/**
1331 * cik_asic_pci_config_reset - soft reset GPU
1332 *
1333 * @adev: amdgpu_device pointer
1334 *
1335 * Use PCI Config method to reset the GPU.
1336 *
1337 * Returns 0 for success.
1338 */
1339static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340{
1341	struct kv_reset_save_regs kv_save = { 0 };
1342	u32 i;
1343	int r = -EINVAL;
1344
1345	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347	if (adev->flags & AMD_IS_APU)
1348		kv_save_regs_for_reset(adev, &kv_save);
1349
1350	/* disable BM */
1351	pci_clear_master(adev->pdev);
1352	/* reset */
1353	amdgpu_device_pci_config_reset(adev);
1354
1355	udelay(100);
1356
1357	/* wait for asic to come out of reset */
1358	for (i = 0; i < adev->usec_timeout; i++) {
1359		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360			/* enable BM */
1361			pci_set_master(adev->pdev);
1362			adev->has_hw_reset = true;
1363			r = 0;
1364			break;
1365		}
1366		udelay(1);
1367	}
1368
1369	/* does asic init need to be run first??? */
1370	if (adev->flags & AMD_IS_APU)
1371		kv_restore_regs_for_reset(adev, &kv_save);
1372
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1373	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375	return r;
1376}
1377
1378static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379{
1380	switch (adev->asic_type) {
1381	case CHIP_BONAIRE:
1382	case CHIP_HAWAII:
1383		return amdgpu_dpm_is_baco_supported(adev);
1384	default:
1385		return false;
1386	}
1387}
1388
1389static enum amd_reset_method
1390cik_asic_reset_method(struct amdgpu_device *adev)
1391{
1392	bool baco_reset;
1393
1394	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396		return amdgpu_reset_method;
1397
1398	if (amdgpu_reset_method != -1)
1399		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400				  amdgpu_reset_method);
1401
1402	switch (adev->asic_type) {
1403	case CHIP_BONAIRE:
1404	case CHIP_HAWAII:
1405		baco_reset = cik_asic_supports_baco(adev);
 
 
1406		break;
1407	default:
1408		baco_reset = false;
1409		break;
1410	}
1411
1412	if (baco_reset)
1413		return AMD_RESET_METHOD_BACO;
1414	else
1415		return AMD_RESET_METHOD_LEGACY;
1416}
1417
1418/**
1419 * cik_asic_reset - soft reset GPU
1420 *
1421 * @adev: amdgpu_device pointer
1422 *
1423 * Look up which blocks are hung and attempt
1424 * to reset them.
1425 * Returns 0 for success.
1426 */
1427static int cik_asic_reset(struct amdgpu_device *adev)
1428{
1429	int r;
1430
1431	/* APUs don't have full asic reset */
1432	if (adev->flags & AMD_IS_APU)
1433		return 0;
1434
1435	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436		dev_info(adev->dev, "BACO reset\n");
1437		r = amdgpu_dpm_baco_reset(adev);
1438	} else {
1439		dev_info(adev->dev, "PCI CONFIG reset\n");
1440		r = cik_asic_pci_config_reset(adev);
1441	}
1442
1443	return r;
1444}
1445
1446static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447{
1448	return RREG32(mmCONFIG_MEMSIZE);
1449}
1450
1451static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452			      u32 cntl_reg, u32 status_reg)
1453{
1454	int r, i;
1455	struct atom_clock_dividers dividers;
1456	uint32_t tmp;
1457
1458	r = amdgpu_atombios_get_clock_dividers(adev,
1459					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460					       clock, false, &dividers);
1461	if (r)
1462		return r;
1463
1464	tmp = RREG32_SMC(cntl_reg);
1465	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467	tmp |= dividers.post_divider;
1468	WREG32_SMC(cntl_reg, tmp);
1469
1470	for (i = 0; i < 100; i++) {
1471		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472			break;
1473		mdelay(10);
1474	}
1475	if (i == 100)
1476		return -ETIMEDOUT;
1477
1478	return 0;
1479}
1480
1481static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482{
1483	int r = 0;
1484
1485	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486	if (r)
1487		return r;
1488
1489	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490	return r;
1491}
1492
1493static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494{
1495	int r, i;
1496	struct atom_clock_dividers dividers;
1497	u32 tmp;
1498
1499	r = amdgpu_atombios_get_clock_dividers(adev,
1500					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501					       ecclk, false, &dividers);
1502	if (r)
1503		return r;
1504
1505	for (i = 0; i < 100; i++) {
1506		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507			break;
1508		mdelay(10);
1509	}
1510	if (i == 100)
1511		return -ETIMEDOUT;
1512
1513	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516	tmp |= dividers.post_divider;
1517	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518
1519	for (i = 0; i < 100; i++) {
1520		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521			break;
1522		mdelay(10);
1523	}
1524	if (i == 100)
1525		return -ETIMEDOUT;
1526
1527	return 0;
1528}
1529
1530static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531{
1532	struct pci_dev *root = adev->pdev->bus->self;
1533	u32 speed_cntl, current_data_rate;
1534	int i;
1535	u16 tmp16;
1536
1537	if (pci_is_root_bus(adev->pdev->bus))
1538		return;
1539
1540	if (amdgpu_pcie_gen2 == 0)
1541		return;
1542
1543	if (adev->flags & AMD_IS_APU)
1544		return;
1545
1546	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548		return;
1549
1550	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554		if (current_data_rate == 2) {
1555			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556			return;
1557		}
1558		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560		if (current_data_rate == 1) {
1561			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562			return;
1563		}
1564		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565	}
1566
1567	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568		return;
1569
1570	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571		/* re-try equalization if gen3 is not already enabled */
1572		if (current_data_rate != 2) {
1573			u16 bridge_cfg, gpu_cfg;
1574			u16 bridge_cfg2, gpu_cfg2;
1575			u32 max_lw, current_lw, tmp;
1576
1577			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
 
 
 
1579
1580			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585
1586			if (current_lw < max_lw) {
1587				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591					tmp |= (max_lw <<
1592						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597				}
1598			}
1599
1600			for (i = 0; i < 10; i++) {
1601				/* check status */
1602				pcie_capability_read_word(adev->pdev,
1603							  PCI_EXP_DEVSTA,
1604							  &tmp16);
1605				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606					break;
1607
1608				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609							  &bridge_cfg);
1610				pcie_capability_read_word(adev->pdev,
1611							  PCI_EXP_LNKCTL,
1612							  &gpu_cfg);
1613
1614				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615							  &bridge_cfg2);
1616				pcie_capability_read_word(adev->pdev,
1617							  PCI_EXP_LNKCTL2,
1618							  &gpu_cfg2);
1619
1620				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623
1624				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627
1628				msleep(100);
1629
1630				/* linkctl */
1631				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632								   PCI_EXP_LNKCTL_HAWD,
1633								   bridge_cfg &
1634								   PCI_EXP_LNKCTL_HAWD);
1635				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636								   PCI_EXP_LNKCTL_HAWD,
1637								   gpu_cfg &
1638								   PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
 
1639
1640				/* linkctl2 */
1641				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
1642								   PCI_EXP_LNKCTL2_ENTER_COMP |
1643								   PCI_EXP_LNKCTL2_TX_MARGIN,
1644								   bridge_cfg2 &
1645								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1646								    PCI_EXP_LNKCTL2_TX_MARGIN));
1647				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1648								   PCI_EXP_LNKCTL2_ENTER_COMP |
1649								   PCI_EXP_LNKCTL2_TX_MARGIN,
1650								   gpu_cfg2 &
1651								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1652								    PCI_EXP_LNKCTL2_TX_MARGIN));
 
 
 
 
 
 
 
 
 
 
1653
1654				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1655				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1656				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1657			}
1658		}
1659	}
1660
1661	/* set the link speed */
1662	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1663		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1664	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1665	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1666
1667	tmp16 = 0;
 
 
1668	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1669		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1670	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1671		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1672	else
1673		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1674	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1675					   PCI_EXP_LNKCTL2_TLS, tmp16);
1676
1677	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1678	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1679	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1680
1681	for (i = 0; i < adev->usec_timeout; i++) {
1682		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1683		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1684			break;
1685		udelay(1);
1686	}
1687}
1688
1689static void cik_program_aspm(struct amdgpu_device *adev)
1690{
1691	u32 data, orig;
1692	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1693	bool disable_clkreq = false;
1694
1695	if (!amdgpu_device_should_use_aspm(adev))
1696		return;
1697
1698	if (pci_is_root_bus(adev->pdev->bus))
1699		return;
1700
 
 
 
 
1701	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1702	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1703	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1704		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1705	if (orig != data)
1706		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1707
1708	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1709	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1710	if (orig != data)
1711		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1712
1713	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1714	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1715	if (orig != data)
1716		WREG32_PCIE(ixPCIE_P_CNTL, data);
1717
1718	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1719	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1720		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1721	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1722	if (!disable_l0s)
1723		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1724
1725	if (!disable_l1) {
1726		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1727		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1728		if (orig != data)
1729			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1730
1731		if (!disable_plloff_in_l1) {
1732			bool clk_req_support;
1733
1734			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1735			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1736				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1737			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1738				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1739			if (orig != data)
1740				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1741
1742			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1743			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1744				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1745			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1746				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1747			if (orig != data)
1748				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1749
1750			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1751			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1752				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1753			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1754				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1755			if (orig != data)
1756				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1757
1758			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1759			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1760				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1761			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1762				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1763			if (orig != data)
1764				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1765
1766			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1767			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1768			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1769			if (orig != data)
1770				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1771
1772			if (!disable_clkreq) {
1773				struct pci_dev *root = adev->pdev->bus->self;
1774				u32 lnkcap;
1775
1776				clk_req_support = false;
1777				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1778				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1779					clk_req_support = true;
1780			} else {
1781				clk_req_support = false;
1782			}
1783
1784			if (clk_req_support) {
1785				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1786				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1787					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1788				if (orig != data)
1789					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1790
1791				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1792				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1793					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1794				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1795					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1796				if (orig != data)
1797					WREG32_SMC(ixTHM_CLK_CNTL, data);
1798
1799				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1800				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1801					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1802				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1803					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1804				if (orig != data)
1805					WREG32_SMC(ixMISC_CLK_CTRL, data);
1806
1807				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1808				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1809				if (orig != data)
1810					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1811
1812				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1813				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1814				if (orig != data)
1815					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1816
1817				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1818				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1819				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1820				if (orig != data)
1821					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1822			}
1823		}
1824	} else {
1825		if (orig != data)
1826			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1827	}
1828
1829	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1830	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1831		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1832		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1833	if (orig != data)
1834		WREG32_PCIE(ixPCIE_CNTL2, data);
1835
1836	if (!disable_l0s) {
1837		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1838		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1839				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1840			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1841			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1842			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1843				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1844				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1845				if (orig != data)
1846					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1847			}
1848		}
1849	}
1850}
1851
1852static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1853{
1854	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1855		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1856}
1857
1858static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1859{
1860	if (!ring || !ring->funcs->emit_wreg) {
1861		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1862		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1863	} else {
1864		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1865	}
1866}
1867
1868static void cik_invalidate_hdp(struct amdgpu_device *adev,
1869			       struct amdgpu_ring *ring)
1870{
1871	if (!ring || !ring->funcs->emit_wreg) {
1872		WREG32(mmHDP_DEBUG0, 1);
1873		RREG32(mmHDP_DEBUG0);
1874	} else {
1875		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1876	}
1877}
1878
1879static bool cik_need_full_reset(struct amdgpu_device *adev)
1880{
1881	/* change this when we support soft reset */
1882	return true;
1883}
1884
1885static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1886			       uint64_t *count1)
1887{
1888	uint32_t perfctr = 0;
1889	uint64_t cnt0_of, cnt1_of;
1890	int tmp;
1891
1892	/* This reports 0 on APUs, so return to avoid writing/reading registers
1893	 * that may or may not be different from their GPU counterparts
1894	 */
1895	if (adev->flags & AMD_IS_APU)
1896		return;
1897
1898	/* Set the 2 events that we wish to watch, defined above */
1899	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1900	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1901	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1902
1903	/* Write to enable desired perf counters */
1904	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1905	/* Zero out and enable the perf counters
1906	 * Write 0x5:
1907	 * Bit 0 = Start all counters(1)
1908	 * Bit 2 = Global counter reset enable(1)
1909	 */
1910	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1911
1912	msleep(1000);
1913
1914	/* Load the shadow and disable the perf counters
1915	 * Write 0x2:
1916	 * Bit 0 = Stop counters(0)
1917	 * Bit 1 = Load the shadow counters(1)
1918	 */
1919	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1920
1921	/* Read register values to get any >32bit overflow */
1922	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1923	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1924	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1925
1926	/* Get the values and add the overflow */
1927	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1928	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1929}
1930
1931static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1932{
1933	u32 clock_cntl, pc;
1934
1935	if (adev->flags & AMD_IS_APU)
1936		return false;
1937
1938	/* check if the SMC is already running */
1939	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1940	pc = RREG32_SMC(ixSMC_PC_C);
1941	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1942	    (0x20100 <= pc))
1943		return true;
1944
1945	return false;
1946}
1947
1948static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1949{
1950	uint64_t nak_r, nak_g;
1951
1952	/* Get the number of NAKs received and generated */
1953	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1954	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1955
1956	/* Add the total number of NAKs, i.e the number of replays */
1957	return (nak_r + nak_g);
1958}
1959
1960static void cik_pre_asic_init(struct amdgpu_device *adev)
1961{
1962}
1963
1964static const struct amdgpu_asic_funcs cik_asic_funcs =
1965{
1966	.read_disabled_bios = &cik_read_disabled_bios,
1967	.read_bios_from_rom = &cik_read_bios_from_rom,
1968	.read_register = &cik_read_register,
1969	.reset = &cik_asic_reset,
1970	.reset_method = &cik_asic_reset_method,
1971	.set_vga_state = &cik_vga_set_state,
1972	.get_xclk = &cik_get_xclk,
1973	.set_uvd_clocks = &cik_set_uvd_clocks,
1974	.set_vce_clocks = &cik_set_vce_clocks,
1975	.get_config_memsize = &cik_get_config_memsize,
1976	.flush_hdp = &cik_flush_hdp,
1977	.invalidate_hdp = &cik_invalidate_hdp,
1978	.need_full_reset = &cik_need_full_reset,
1979	.init_doorbell_index = &legacy_doorbell_index_init,
1980	.get_pcie_usage = &cik_get_pcie_usage,
1981	.need_reset_on_init = &cik_need_reset_on_init,
1982	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1983	.supports_baco = &cik_asic_supports_baco,
1984	.pre_asic_init = &cik_pre_asic_init,
1985	.query_video_codecs = &cik_query_video_codecs,
1986};
1987
1988static int cik_common_early_init(void *handle)
1989{
1990	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1991
1992	adev->smc_rreg = &cik_smc_rreg;
1993	adev->smc_wreg = &cik_smc_wreg;
1994	adev->pcie_rreg = &cik_pcie_rreg;
1995	adev->pcie_wreg = &cik_pcie_wreg;
1996	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1997	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1998	adev->didt_rreg = &cik_didt_rreg;
1999	adev->didt_wreg = &cik_didt_wreg;
2000
2001	adev->asic_funcs = &cik_asic_funcs;
2002
2003	adev->rev_id = cik_get_rev_id(adev);
2004	adev->external_rev_id = 0xFF;
2005	switch (adev->asic_type) {
2006	case CHIP_BONAIRE:
2007		adev->cg_flags =
2008			AMD_CG_SUPPORT_GFX_MGCG |
2009			AMD_CG_SUPPORT_GFX_MGLS |
2010			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2011			AMD_CG_SUPPORT_GFX_CGLS |
2012			AMD_CG_SUPPORT_GFX_CGTS |
2013			AMD_CG_SUPPORT_GFX_CGTS_LS |
2014			AMD_CG_SUPPORT_GFX_CP_LS |
2015			AMD_CG_SUPPORT_MC_LS |
2016			AMD_CG_SUPPORT_MC_MGCG |
2017			AMD_CG_SUPPORT_SDMA_MGCG |
2018			AMD_CG_SUPPORT_SDMA_LS |
2019			AMD_CG_SUPPORT_BIF_LS |
2020			AMD_CG_SUPPORT_VCE_MGCG |
2021			AMD_CG_SUPPORT_UVD_MGCG |
2022			AMD_CG_SUPPORT_HDP_LS |
2023			AMD_CG_SUPPORT_HDP_MGCG;
2024		adev->pg_flags = 0;
2025		adev->external_rev_id = adev->rev_id + 0x14;
2026		break;
2027	case CHIP_HAWAII:
2028		adev->cg_flags =
2029			AMD_CG_SUPPORT_GFX_MGCG |
2030			AMD_CG_SUPPORT_GFX_MGLS |
2031			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2032			AMD_CG_SUPPORT_GFX_CGLS |
2033			AMD_CG_SUPPORT_GFX_CGTS |
2034			AMD_CG_SUPPORT_GFX_CP_LS |
2035			AMD_CG_SUPPORT_MC_LS |
2036			AMD_CG_SUPPORT_MC_MGCG |
2037			AMD_CG_SUPPORT_SDMA_MGCG |
2038			AMD_CG_SUPPORT_SDMA_LS |
2039			AMD_CG_SUPPORT_BIF_LS |
2040			AMD_CG_SUPPORT_VCE_MGCG |
2041			AMD_CG_SUPPORT_UVD_MGCG |
2042			AMD_CG_SUPPORT_HDP_LS |
2043			AMD_CG_SUPPORT_HDP_MGCG;
2044		adev->pg_flags = 0;
2045		adev->external_rev_id = 0x28;
2046		break;
2047	case CHIP_KAVERI:
2048		adev->cg_flags =
2049			AMD_CG_SUPPORT_GFX_MGCG |
2050			AMD_CG_SUPPORT_GFX_MGLS |
2051			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2052			AMD_CG_SUPPORT_GFX_CGLS |
2053			AMD_CG_SUPPORT_GFX_CGTS |
2054			AMD_CG_SUPPORT_GFX_CGTS_LS |
2055			AMD_CG_SUPPORT_GFX_CP_LS |
2056			AMD_CG_SUPPORT_SDMA_MGCG |
2057			AMD_CG_SUPPORT_SDMA_LS |
2058			AMD_CG_SUPPORT_BIF_LS |
2059			AMD_CG_SUPPORT_VCE_MGCG |
2060			AMD_CG_SUPPORT_UVD_MGCG |
2061			AMD_CG_SUPPORT_HDP_LS |
2062			AMD_CG_SUPPORT_HDP_MGCG;
2063		adev->pg_flags =
2064			/*AMD_PG_SUPPORT_GFX_PG |
2065			  AMD_PG_SUPPORT_GFX_SMG |
2066			  AMD_PG_SUPPORT_GFX_DMG |*/
2067			AMD_PG_SUPPORT_UVD |
2068			AMD_PG_SUPPORT_VCE |
2069			/*  AMD_PG_SUPPORT_CP |
2070			  AMD_PG_SUPPORT_GDS |
2071			  AMD_PG_SUPPORT_RLC_SMU_HS |
2072			  AMD_PG_SUPPORT_ACP |
2073			  AMD_PG_SUPPORT_SAMU |*/
2074			0;
2075		if (adev->pdev->device == 0x1312 ||
2076			adev->pdev->device == 0x1316 ||
2077			adev->pdev->device == 0x1317)
2078			adev->external_rev_id = 0x41;
2079		else
2080			adev->external_rev_id = 0x1;
2081		break;
2082	case CHIP_KABINI:
2083	case CHIP_MULLINS:
2084		adev->cg_flags =
2085			AMD_CG_SUPPORT_GFX_MGCG |
2086			AMD_CG_SUPPORT_GFX_MGLS |
2087			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2088			AMD_CG_SUPPORT_GFX_CGLS |
2089			AMD_CG_SUPPORT_GFX_CGTS |
2090			AMD_CG_SUPPORT_GFX_CGTS_LS |
2091			AMD_CG_SUPPORT_GFX_CP_LS |
2092			AMD_CG_SUPPORT_SDMA_MGCG |
2093			AMD_CG_SUPPORT_SDMA_LS |
2094			AMD_CG_SUPPORT_BIF_LS |
2095			AMD_CG_SUPPORT_VCE_MGCG |
2096			AMD_CG_SUPPORT_UVD_MGCG |
2097			AMD_CG_SUPPORT_HDP_LS |
2098			AMD_CG_SUPPORT_HDP_MGCG;
2099		adev->pg_flags =
2100			/*AMD_PG_SUPPORT_GFX_PG |
2101			  AMD_PG_SUPPORT_GFX_SMG | */
2102			AMD_PG_SUPPORT_UVD |
2103			/*AMD_PG_SUPPORT_VCE |
2104			  AMD_PG_SUPPORT_CP |
2105			  AMD_PG_SUPPORT_GDS |
2106			  AMD_PG_SUPPORT_RLC_SMU_HS |
2107			  AMD_PG_SUPPORT_SAMU |*/
2108			0;
2109		if (adev->asic_type == CHIP_KABINI) {
2110			if (adev->rev_id == 0)
2111				adev->external_rev_id = 0x81;
2112			else if (adev->rev_id == 1)
2113				adev->external_rev_id = 0x82;
2114			else if (adev->rev_id == 2)
2115				adev->external_rev_id = 0x85;
2116		} else
2117			adev->external_rev_id = adev->rev_id + 0xa1;
2118		break;
2119	default:
2120		/* FIXME: not supported yet */
2121		return -EINVAL;
2122	}
2123
2124	return 0;
2125}
2126
2127static int cik_common_sw_init(void *handle)
2128{
2129	return 0;
2130}
2131
2132static int cik_common_sw_fini(void *handle)
2133{
2134	return 0;
2135}
2136
2137static int cik_common_hw_init(void *handle)
2138{
2139	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2140
2141	/* move the golden regs per IP block */
2142	cik_init_golden_registers(adev);
2143	/* enable pcie gen2/3 link */
2144	cik_pcie_gen3_enable(adev);
2145	/* enable aspm */
2146	cik_program_aspm(adev);
2147
2148	return 0;
2149}
2150
2151static int cik_common_hw_fini(void *handle)
2152{
2153	return 0;
2154}
2155
2156static int cik_common_suspend(void *handle)
2157{
2158	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2159
2160	return cik_common_hw_fini(adev);
2161}
2162
2163static int cik_common_resume(void *handle)
2164{
2165	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2166
2167	return cik_common_hw_init(adev);
2168}
2169
2170static bool cik_common_is_idle(void *handle)
2171{
2172	return true;
2173}
2174
2175static int cik_common_wait_for_idle(void *handle)
2176{
2177	return 0;
2178}
2179
2180static int cik_common_soft_reset(void *handle)
2181{
2182	/* XXX hard reset?? */
2183	return 0;
2184}
2185
2186static int cik_common_set_clockgating_state(void *handle,
2187					    enum amd_clockgating_state state)
2188{
2189	return 0;
2190}
2191
2192static int cik_common_set_powergating_state(void *handle,
2193					    enum amd_powergating_state state)
2194{
2195	return 0;
2196}
2197
2198static const struct amd_ip_funcs cik_common_ip_funcs = {
2199	.name = "cik_common",
2200	.early_init = cik_common_early_init,
2201	.late_init = NULL,
2202	.sw_init = cik_common_sw_init,
2203	.sw_fini = cik_common_sw_fini,
2204	.hw_init = cik_common_hw_init,
2205	.hw_fini = cik_common_hw_fini,
2206	.suspend = cik_common_suspend,
2207	.resume = cik_common_resume,
2208	.is_idle = cik_common_is_idle,
2209	.wait_for_idle = cik_common_wait_for_idle,
2210	.soft_reset = cik_common_soft_reset,
2211	.set_clockgating_state = cik_common_set_clockgating_state,
2212	.set_powergating_state = cik_common_set_powergating_state,
2213};
2214
2215static const struct amdgpu_ip_block_version cik_common_ip_block =
2216{
2217	.type = AMD_IP_BLOCK_TYPE_COMMON,
2218	.major = 1,
2219	.minor = 0,
2220	.rev = 0,
2221	.funcs = &cik_common_ip_funcs,
2222};
2223
2224int cik_set_ip_blocks(struct amdgpu_device *adev)
2225{
2226	switch (adev->asic_type) {
2227	case CHIP_BONAIRE:
2228		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2229		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2230		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2231		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2232		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2233		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2234		if (adev->enable_virtual_display)
2235			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2236#if defined(CONFIG_DRM_AMD_DC)
2237		else if (amdgpu_device_has_dc_support(adev))
2238			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2239#endif
2240		else
2241			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2242		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2243		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2244		break;
2245	case CHIP_HAWAII:
2246		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2247		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2248		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2249		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2250		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2251		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2252		if (adev->enable_virtual_display)
2253			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2254#if defined(CONFIG_DRM_AMD_DC)
2255		else if (amdgpu_device_has_dc_support(adev))
2256			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2257#endif
2258		else
2259			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2260		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2261		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2262		break;
2263	case CHIP_KAVERI:
2264		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2265		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2266		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2267		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2268		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2269		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2270		if (adev->enable_virtual_display)
2271			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2272#if defined(CONFIG_DRM_AMD_DC)
2273		else if (amdgpu_device_has_dc_support(adev))
2274			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2275#endif
2276		else
2277			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2278
2279		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2280		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2281		break;
2282	case CHIP_KABINI:
2283	case CHIP_MULLINS:
2284		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2285		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2286		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2287		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2288		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2289		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2290		if (adev->enable_virtual_display)
2291			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2292#if defined(CONFIG_DRM_AMD_DC)
2293		else if (amdgpu_device_has_dc_support(adev))
2294			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2295#endif
2296		else
2297			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2298		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2299		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2300		break;
2301	default:
2302		/* FIXME: not supported yet */
2303		return -EINVAL;
2304	}
2305	return 0;
2306}