1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include <drm/amdgpu_drm.h>
  30
  31#include "amdgpu.h"
  32#include "amdgpu_atombios.h"
  33#include "amdgpu_ih.h"
  34#include "amdgpu_uvd.h"
  35#include "amdgpu_vce.h"
  36#include "cikd.h"
  37#include "atom.h"
  38#include "amd_pcie.h"
  39
  40#include "cik.h"
  41#include "gmc_v7_0.h"
  42#include "cik_ih.h"
  43#include "dce_v8_0.h"
  44#include "gfx_v7_0.h"
  45#include "cik_sdma.h"
  46#include "uvd_v4_2.h"
  47#include "vce_v2_0.h"
  48#include "cik_dpm.h"
  49
  50#include "uvd/uvd_4_2_d.h"
  51
  52#include "smu/smu_7_0_1_d.h"
  53#include "smu/smu_7_0_1_sh_mask.h"
  54
  55#include "dce/dce_8_0_d.h"
  56#include "dce/dce_8_0_sh_mask.h"
  57
  58#include "bif/bif_4_1_d.h"
  59#include "bif/bif_4_1_sh_mask.h"
  60
  61#include "gca/gfx_7_2_d.h"
  62#include "gca/gfx_7_2_enum.h"
  63#include "gca/gfx_7_2_sh_mask.h"
  64
  65#include "gmc/gmc_7_1_d.h"
  66#include "gmc/gmc_7_1_sh_mask.h"
  67
  68#include "oss/oss_2_0_d.h"
  69#include "oss/oss_2_0_sh_mask.h"
  70
  71#include "amdgpu_dm.h"
  72#include "amdgpu_amdkfd.h"
  73#include "amdgpu_vkms.h"
  74
  75static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
  76{
  77	{
  78		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
  79		.max_width = 2048,
  80		.max_height = 1152,
  81		.max_pixels_per_frame = 2048 * 1152,
  82		.max_level = 0,
  83	},
  84};
  85
  86static const struct amdgpu_video_codecs cik_video_codecs_encode =
  87{
  88	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
  89	.codec_array = cik_video_codecs_encode_array,
  90};
  91
  92static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
  93{
  94	{
  95		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
  96		.max_width = 2048,
  97		.max_height = 1152,
  98		.max_pixels_per_frame = 2048 * 1152,
  99		.max_level = 3,
 100	},
 101	{
 102		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 103		.max_width = 2048,
 104		.max_height = 1152,
 105		.max_pixels_per_frame = 2048 * 1152,
 106		.max_level = 5,
 107	},
 108	{
 109		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 110		.max_width = 2048,
 111		.max_height = 1152,
 112		.max_pixels_per_frame = 2048 * 1152,
 113		.max_level = 41,
 114	},
 115	{
 116		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 117		.max_width = 2048,
 118		.max_height = 1152,
 119		.max_pixels_per_frame = 2048 * 1152,
 120		.max_level = 4,
 121	},
 122};
 123
 124static const struct amdgpu_video_codecs cik_video_codecs_decode =
 125{
 126	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
 127	.codec_array = cik_video_codecs_decode_array,
 128};
 129
 130static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
 131				  const struct amdgpu_video_codecs **codecs)
 132{
 133	switch (adev->asic_type) {
 134	case CHIP_BONAIRE:
 135	case CHIP_HAWAII:
 136	case CHIP_KAVERI:
 137	case CHIP_KABINI:
 138	case CHIP_MULLINS:
 139		if (encode)
 140			*codecs = &cik_video_codecs_encode;
 141		else
 142			*codecs = &cik_video_codecs_decode;
 143		return 0;
 144	default:
 145		return -EINVAL;
 146	}
 147}
 148
 149/*
 150 * Indirect registers accessor
 151 */
 152static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 153{
 154	unsigned long flags;
 155	u32 r;
 156
 157	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 158	WREG32(mmPCIE_INDEX, reg);
 159	(void)RREG32(mmPCIE_INDEX);
 160	r = RREG32(mmPCIE_DATA);
 161	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162	return r;
 163}
 164
 165static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 166{
 167	unsigned long flags;
 168
 169	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 170	WREG32(mmPCIE_INDEX, reg);
 171	(void)RREG32(mmPCIE_INDEX);
 172	WREG32(mmPCIE_DATA, v);
 173	(void)RREG32(mmPCIE_DATA);
 174	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 175}
 176
 177static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 178{
 179	unsigned long flags;
 180	u32 r;
 181
 182	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 183	WREG32(mmSMC_IND_INDEX_0, (reg));
 184	r = RREG32(mmSMC_IND_DATA_0);
 185	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 186	return r;
 187}
 188
 189static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 190{
 191	unsigned long flags;
 192
 193	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 194	WREG32(mmSMC_IND_INDEX_0, (reg));
 195	WREG32(mmSMC_IND_DATA_0, (v));
 196	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 197}
 198
 199static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 200{
 201	unsigned long flags;
 202	u32 r;
 203
 204	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 205	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 206	r = RREG32(mmUVD_CTX_DATA);
 207	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 208	return r;
 209}
 210
 211static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 212{
 213	unsigned long flags;
 214
 215	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 216	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 217	WREG32(mmUVD_CTX_DATA, (v));
 218	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 219}
 220
 221static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 222{
 223	unsigned long flags;
 224	u32 r;
 225
 226	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 227	WREG32(mmDIDT_IND_INDEX, (reg));
 228	r = RREG32(mmDIDT_IND_DATA);
 229	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 230	return r;
 231}
 232
 233static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 234{
 235	unsigned long flags;
 236
 237	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 238	WREG32(mmDIDT_IND_INDEX, (reg));
 239	WREG32(mmDIDT_IND_DATA, (v));
 240	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 241}
 242
 243static const u32 bonaire_golden_spm_registers[] =
 244{
 245	0xc200, 0xe0ffffff, 0xe0000000
 246};
 247
 248static const u32 bonaire_golden_common_registers[] =
 249{
 250	0x31dc, 0xffffffff, 0x00000800,
 251	0x31dd, 0xffffffff, 0x00000800,
 252	0x31e6, 0xffffffff, 0x00007fbf,
 253	0x31e7, 0xffffffff, 0x00007faf
 254};
 255
 256static const u32 bonaire_golden_registers[] =
 257{
 258	0xcd5, 0x00000333, 0x00000333,
 259	0xcd4, 0x000c0fc0, 0x00040200,
 260	0x2684, 0x00010000, 0x00058208,
 261	0xf000, 0xffff1fff, 0x00140000,
 262	0xf080, 0xfdfc0fff, 0x00000100,
 263	0xf08d, 0x40000000, 0x40000200,
 264	0x260c, 0xffffffff, 0x00000000,
 265	0x260d, 0xf00fffff, 0x00000400,
 266	0x260e, 0x0002021c, 0x00020200,
 267	0x31e, 0x00000080, 0x00000000,
 268	0x16ec, 0x000000f0, 0x00000070,
 269	0x16f0, 0xf0311fff, 0x80300000,
 270	0x263e, 0x73773777, 0x12010001,
 271	0xd43, 0x00810000, 0x408af000,
 272	0x1c0c, 0x31000111, 0x00000011,
 273	0xbd2, 0x73773777, 0x12010001,
 274	0x883, 0x00007fb6, 0x0021a1b1,
 275	0x884, 0x00007fb6, 0x002021b1,
 276	0x860, 0x00007fb6, 0x00002191,
 277	0x886, 0x00007fb6, 0x002121b1,
 278	0x887, 0x00007fb6, 0x002021b1,
 279	0x877, 0x00007fb6, 0x00002191,
 280	0x878, 0x00007fb6, 0x00002191,
 281	0xd8a, 0x0000003f, 0x0000000a,
 282	0xd8b, 0x0000003f, 0x0000000a,
 283	0xab9, 0x00073ffe, 0x000022a2,
 284	0x903, 0x000007ff, 0x00000000,
 285	0x2285, 0xf000003f, 0x00000007,
 286	0x22fc, 0x00002001, 0x00000001,
 287	0x22c9, 0xffffffff, 0x00ffffff,
 288	0xc281, 0x0000ff0f, 0x00000000,
 289	0xa293, 0x07ffffff, 0x06000000,
 290	0x136, 0x00000fff, 0x00000100,
 291	0xf9e, 0x00000001, 0x00000002,
 292	0x2440, 0x03000000, 0x0362c688,
 293	0x2300, 0x000000ff, 0x00000001,
 294	0x390, 0x00001fff, 0x00001fff,
 295	0x2418, 0x0000007f, 0x00000020,
 296	0x2542, 0x00010000, 0x00010000,
 297	0x2b05, 0x000003ff, 0x000000f3,
 298	0x2b03, 0xffffffff, 0x00001032
 299};
 300
 301static const u32 bonaire_mgcg_cgcg_init[] =
 302{
 303	0x3108, 0xffffffff, 0xfffffffc,
 304	0xc200, 0xffffffff, 0xe0000000,
 305	0xf0a8, 0xffffffff, 0x00000100,
 306	0xf082, 0xffffffff, 0x00000100,
 307	0xf0b0, 0xffffffff, 0xc0000100,
 308	0xf0b2, 0xffffffff, 0xc0000100,
 309	0xf0b1, 0xffffffff, 0xc0000100,
 310	0x1579, 0xffffffff, 0x00600100,
 311	0xf0a0, 0xffffffff, 0x00000100,
 312	0xf085, 0xffffffff, 0x06000100,
 313	0xf088, 0xffffffff, 0x00000100,
 314	0xf086, 0xffffffff, 0x06000100,
 315	0xf081, 0xffffffff, 0x00000100,
 316	0xf0b8, 0xffffffff, 0x00000100,
 317	0xf089, 0xffffffff, 0x00000100,
 318	0xf080, 0xffffffff, 0x00000100,
 319	0xf08c, 0xffffffff, 0x00000100,
 320	0xf08d, 0xffffffff, 0x00000100,
 321	0xf094, 0xffffffff, 0x00000100,
 322	0xf095, 0xffffffff, 0x00000100,
 323	0xf096, 0xffffffff, 0x00000100,
 324	0xf097, 0xffffffff, 0x00000100,
 325	0xf098, 0xffffffff, 0x00000100,
 326	0xf09f, 0xffffffff, 0x00000100,
 327	0xf09e, 0xffffffff, 0x00000100,
 328	0xf084, 0xffffffff, 0x06000100,
 329	0xf0a4, 0xffffffff, 0x00000100,
 330	0xf09d, 0xffffffff, 0x00000100,
 331	0xf0ad, 0xffffffff, 0x00000100,
 332	0xf0ac, 0xffffffff, 0x00000100,
 333	0xf09c, 0xffffffff, 0x00000100,
 334	0xc200, 0xffffffff, 0xe0000000,
 335	0xf008, 0xffffffff, 0x00010000,
 336	0xf009, 0xffffffff, 0x00030002,
 337	0xf00a, 0xffffffff, 0x00040007,
 338	0xf00b, 0xffffffff, 0x00060005,
 339	0xf00c, 0xffffffff, 0x00090008,
 340	0xf00d, 0xffffffff, 0x00010000,
 341	0xf00e, 0xffffffff, 0x00030002,
 342	0xf00f, 0xffffffff, 0x00040007,
 343	0xf010, 0xffffffff, 0x00060005,
 344	0xf011, 0xffffffff, 0x00090008,
 345	0xf012, 0xffffffff, 0x00010000,
 346	0xf013, 0xffffffff, 0x00030002,
 347	0xf014, 0xffffffff, 0x00040007,
 348	0xf015, 0xffffffff, 0x00060005,
 349	0xf016, 0xffffffff, 0x00090008,
 350	0xf017, 0xffffffff, 0x00010000,
 351	0xf018, 0xffffffff, 0x00030002,
 352	0xf019, 0xffffffff, 0x00040007,
 353	0xf01a, 0xffffffff, 0x00060005,
 354	0xf01b, 0xffffffff, 0x00090008,
 355	0xf01c, 0xffffffff, 0x00010000,
 356	0xf01d, 0xffffffff, 0x00030002,
 357	0xf01e, 0xffffffff, 0x00040007,
 358	0xf01f, 0xffffffff, 0x00060005,
 359	0xf020, 0xffffffff, 0x00090008,
 360	0xf021, 0xffffffff, 0x00010000,
 361	0xf022, 0xffffffff, 0x00030002,
 362	0xf023, 0xffffffff, 0x00040007,
 363	0xf024, 0xffffffff, 0x00060005,
 364	0xf025, 0xffffffff, 0x00090008,
 365	0xf026, 0xffffffff, 0x00010000,
 366	0xf027, 0xffffffff, 0x00030002,
 367	0xf028, 0xffffffff, 0x00040007,
 368	0xf029, 0xffffffff, 0x00060005,
 369	0xf02a, 0xffffffff, 0x00090008,
 370	0xf000, 0xffffffff, 0x96e00200,
 371	0x21c2, 0xffffffff, 0x00900100,
 372	0x3109, 0xffffffff, 0x0020003f,
 373	0xe, 0xffffffff, 0x0140001c,
 374	0xf, 0x000f0000, 0x000f0000,
 375	0x88, 0xffffffff, 0xc060000c,
 376	0x89, 0xc0000fff, 0x00000100,
 377	0x3e4, 0xffffffff, 0x00000100,
 378	0x3e6, 0x00000101, 0x00000000,
 379	0x82a, 0xffffffff, 0x00000104,
 380	0x1579, 0xff000fff, 0x00000100,
 381	0xc33, 0xc0000fff, 0x00000104,
 382	0x3079, 0x00000001, 0x00000001,
 383	0x3403, 0xff000ff0, 0x00000100,
 384	0x3603, 0xff000ff0, 0x00000100
 385};
 386
 387static const u32 spectre_golden_spm_registers[] =
 388{
 389	0xc200, 0xe0ffffff, 0xe0000000
 390};
 391
 392static const u32 spectre_golden_common_registers[] =
 393{
 394	0x31dc, 0xffffffff, 0x00000800,
 395	0x31dd, 0xffffffff, 0x00000800,
 396	0x31e6, 0xffffffff, 0x00007fbf,
 397	0x31e7, 0xffffffff, 0x00007faf
 398};
 399
 400static const u32 spectre_golden_registers[] =
 401{
 402	0xf000, 0xffff1fff, 0x96940200,
 403	0xf003, 0xffff0001, 0xff000000,
 404	0xf080, 0xfffc0fff, 0x00000100,
 405	0x1bb6, 0x00010101, 0x00010000,
 406	0x260d, 0xf00fffff, 0x00000400,
 407	0x260e, 0xfffffffc, 0x00020200,
 408	0x16ec, 0x000000f0, 0x00000070,
 409	0x16f0, 0xf0311fff, 0x80300000,
 410	0x263e, 0x73773777, 0x12010001,
 411	0x26df, 0x00ff0000, 0x00fc0000,
 412	0xbd2, 0x73773777, 0x12010001,
 413	0x2285, 0xf000003f, 0x00000007,
 414	0x22c9, 0xffffffff, 0x00ffffff,
 415	0xa0d4, 0x3f3f3fff, 0x00000082,
 416	0xa0d5, 0x0000003f, 0x00000000,
 417	0xf9e, 0x00000001, 0x00000002,
 418	0x244f, 0xffff03df, 0x00000004,
 419	0x31da, 0x00000008, 0x00000008,
 420	0x2300, 0x000008ff, 0x00000800,
 421	0x2542, 0x00010000, 0x00010000,
 422	0x2b03, 0xffffffff, 0x54763210,
 423	0x853e, 0x01ff01ff, 0x00000002,
 424	0x8526, 0x007ff800, 0x00200000,
 425	0x8057, 0xffffffff, 0x00000f40,
 426	0xc24d, 0xffffffff, 0x00000001
 427};
 428
 429static const u32 spectre_mgcg_cgcg_init[] =
 430{
 431	0x3108, 0xffffffff, 0xfffffffc,
 432	0xc200, 0xffffffff, 0xe0000000,
 433	0xf0a8, 0xffffffff, 0x00000100,
 434	0xf082, 0xffffffff, 0x00000100,
 435	0xf0b0, 0xffffffff, 0x00000100,
 436	0xf0b2, 0xffffffff, 0x00000100,
 437	0xf0b1, 0xffffffff, 0x00000100,
 438	0x1579, 0xffffffff, 0x00600100,
 439	0xf0a0, 0xffffffff, 0x00000100,
 440	0xf085, 0xffffffff, 0x06000100,
 441	0xf088, 0xffffffff, 0x00000100,
 442	0xf086, 0xffffffff, 0x06000100,
 443	0xf081, 0xffffffff, 0x00000100,
 444	0xf0b8, 0xffffffff, 0x00000100,
 445	0xf089, 0xffffffff, 0x00000100,
 446	0xf080, 0xffffffff, 0x00000100,
 447	0xf08c, 0xffffffff, 0x00000100,
 448	0xf08d, 0xffffffff, 0x00000100,
 449	0xf094, 0xffffffff, 0x00000100,
 450	0xf095, 0xffffffff, 0x00000100,
 451	0xf096, 0xffffffff, 0x00000100,
 452	0xf097, 0xffffffff, 0x00000100,
 453	0xf098, 0xffffffff, 0x00000100,
 454	0xf09f, 0xffffffff, 0x00000100,
 455	0xf09e, 0xffffffff, 0x00000100,
 456	0xf084, 0xffffffff, 0x06000100,
 457	0xf0a4, 0xffffffff, 0x00000100,
 458	0xf09d, 0xffffffff, 0x00000100,
 459	0xf0ad, 0xffffffff, 0x00000100,
 460	0xf0ac, 0xffffffff, 0x00000100,
 461	0xf09c, 0xffffffff, 0x00000100,
 462	0xc200, 0xffffffff, 0xe0000000,
 463	0xf008, 0xffffffff, 0x00010000,
 464	0xf009, 0xffffffff, 0x00030002,
 465	0xf00a, 0xffffffff, 0x00040007,
 466	0xf00b, 0xffffffff, 0x00060005,
 467	0xf00c, 0xffffffff, 0x00090008,
 468	0xf00d, 0xffffffff, 0x00010000,
 469	0xf00e, 0xffffffff, 0x00030002,
 470	0xf00f, 0xffffffff, 0x00040007,
 471	0xf010, 0xffffffff, 0x00060005,
 472	0xf011, 0xffffffff, 0x00090008,
 473	0xf012, 0xffffffff, 0x00010000,
 474	0xf013, 0xffffffff, 0x00030002,
 475	0xf014, 0xffffffff, 0x00040007,
 476	0xf015, 0xffffffff, 0x00060005,
 477	0xf016, 0xffffffff, 0x00090008,
 478	0xf017, 0xffffffff, 0x00010000,
 479	0xf018, 0xffffffff, 0x00030002,
 480	0xf019, 0xffffffff, 0x00040007,
 481	0xf01a, 0xffffffff, 0x00060005,
 482	0xf01b, 0xffffffff, 0x00090008,
 483	0xf01c, 0xffffffff, 0x00010000,
 484	0xf01d, 0xffffffff, 0x00030002,
 485	0xf01e, 0xffffffff, 0x00040007,
 486	0xf01f, 0xffffffff, 0x00060005,
 487	0xf020, 0xffffffff, 0x00090008,
 488	0xf021, 0xffffffff, 0x00010000,
 489	0xf022, 0xffffffff, 0x00030002,
 490	0xf023, 0xffffffff, 0x00040007,
 491	0xf024, 0xffffffff, 0x00060005,
 492	0xf025, 0xffffffff, 0x00090008,
 493	0xf026, 0xffffffff, 0x00010000,
 494	0xf027, 0xffffffff, 0x00030002,
 495	0xf028, 0xffffffff, 0x00040007,
 496	0xf029, 0xffffffff, 0x00060005,
 497	0xf02a, 0xffffffff, 0x00090008,
 498	0xf02b, 0xffffffff, 0x00010000,
 499	0xf02c, 0xffffffff, 0x00030002,
 500	0xf02d, 0xffffffff, 0x00040007,
 501	0xf02e, 0xffffffff, 0x00060005,
 502	0xf02f, 0xffffffff, 0x00090008,
 503	0xf000, 0xffffffff, 0x96e00200,
 504	0x21c2, 0xffffffff, 0x00900100,
 505	0x3109, 0xffffffff, 0x0020003f,
 506	0xe, 0xffffffff, 0x0140001c,
 507	0xf, 0x000f0000, 0x000f0000,
 508	0x88, 0xffffffff, 0xc060000c,
 509	0x89, 0xc0000fff, 0x00000100,
 510	0x3e4, 0xffffffff, 0x00000100,
 511	0x3e6, 0x00000101, 0x00000000,
 512	0x82a, 0xffffffff, 0x00000104,
 513	0x1579, 0xff000fff, 0x00000100,
 514	0xc33, 0xc0000fff, 0x00000104,
 515	0x3079, 0x00000001, 0x00000001,
 516	0x3403, 0xff000ff0, 0x00000100,
 517	0x3603, 0xff000ff0, 0x00000100
 518};
 519
 520static const u32 kalindi_golden_spm_registers[] =
 521{
 522	0xc200, 0xe0ffffff, 0xe0000000
 523};
 524
 525static const u32 kalindi_golden_common_registers[] =
 526{
 527	0x31dc, 0xffffffff, 0x00000800,
 528	0x31dd, 0xffffffff, 0x00000800,
 529	0x31e6, 0xffffffff, 0x00007fbf,
 530	0x31e7, 0xffffffff, 0x00007faf
 531};
 532
 533static const u32 kalindi_golden_registers[] =
 534{
 535	0xf000, 0xffffdfff, 0x6e944040,
 536	0x1579, 0xff607fff, 0xfc000100,
 537	0xf088, 0xff000fff, 0x00000100,
 538	0xf089, 0xff000fff, 0x00000100,
 539	0xf080, 0xfffc0fff, 0x00000100,
 540	0x1bb6, 0x00010101, 0x00010000,
 541	0x260c, 0xffffffff, 0x00000000,
 542	0x260d, 0xf00fffff, 0x00000400,
 543	0x16ec, 0x000000f0, 0x00000070,
 544	0x16f0, 0xf0311fff, 0x80300000,
 545	0x263e, 0x73773777, 0x12010001,
 546	0x263f, 0xffffffff, 0x00000010,
 547	0x26df, 0x00ff0000, 0x00fc0000,
 548	0x200c, 0x00001f0f, 0x0000100a,
 549	0xbd2, 0x73773777, 0x12010001,
 550	0x902, 0x000fffff, 0x000c007f,
 551	0x2285, 0xf000003f, 0x00000007,
 552	0x22c9, 0x3fff3fff, 0x00ffcfff,
 553	0xc281, 0x0000ff0f, 0x00000000,
 554	0xa293, 0x07ffffff, 0x06000000,
 555	0x136, 0x00000fff, 0x00000100,
 556	0xf9e, 0x00000001, 0x00000002,
 557	0x31da, 0x00000008, 0x00000008,
 558	0x2300, 0x000000ff, 0x00000003,
 559	0x853e, 0x01ff01ff, 0x00000002,
 560	0x8526, 0x007ff800, 0x00200000,
 561	0x8057, 0xffffffff, 0x00000f40,
 562	0x2231, 0x001f3ae3, 0x00000082,
 563	0x2235, 0x0000001f, 0x00000010,
 564	0xc24d, 0xffffffff, 0x00000000
 565};
 566
 567static const u32 kalindi_mgcg_cgcg_init[] =
 568{
 569	0x3108, 0xffffffff, 0xfffffffc,
 570	0xc200, 0xffffffff, 0xe0000000,
 571	0xf0a8, 0xffffffff, 0x00000100,
 572	0xf082, 0xffffffff, 0x00000100,
 573	0xf0b0, 0xffffffff, 0x00000100,
 574	0xf0b2, 0xffffffff, 0x00000100,
 575	0xf0b1, 0xffffffff, 0x00000100,
 576	0x1579, 0xffffffff, 0x00600100,
 577	0xf0a0, 0xffffffff, 0x00000100,
 578	0xf085, 0xffffffff, 0x06000100,
 579	0xf088, 0xffffffff, 0x00000100,
 580	0xf086, 0xffffffff, 0x06000100,
 581	0xf081, 0xffffffff, 0x00000100,
 582	0xf0b8, 0xffffffff, 0x00000100,
 583	0xf089, 0xffffffff, 0x00000100,
 584	0xf080, 0xffffffff, 0x00000100,
 585	0xf08c, 0xffffffff, 0x00000100,
 586	0xf08d, 0xffffffff, 0x00000100,
 587	0xf094, 0xffffffff, 0x00000100,
 588	0xf095, 0xffffffff, 0x00000100,
 589	0xf096, 0xffffffff, 0x00000100,
 590	0xf097, 0xffffffff, 0x00000100,
 591	0xf098, 0xffffffff, 0x00000100,
 592	0xf09f, 0xffffffff, 0x00000100,
 593	0xf09e, 0xffffffff, 0x00000100,
 594	0xf084, 0xffffffff, 0x06000100,
 595	0xf0a4, 0xffffffff, 0x00000100,
 596	0xf09d, 0xffffffff, 0x00000100,
 597	0xf0ad, 0xffffffff, 0x00000100,
 598	0xf0ac, 0xffffffff, 0x00000100,
 599	0xf09c, 0xffffffff, 0x00000100,
 600	0xc200, 0xffffffff, 0xe0000000,
 601	0xf008, 0xffffffff, 0x00010000,
 602	0xf009, 0xffffffff, 0x00030002,
 603	0xf00a, 0xffffffff, 0x00040007,
 604	0xf00b, 0xffffffff, 0x00060005,
 605	0xf00c, 0xffffffff, 0x00090008,
 606	0xf00d, 0xffffffff, 0x00010000,
 607	0xf00e, 0xffffffff, 0x00030002,
 608	0xf00f, 0xffffffff, 0x00040007,
 609	0xf010, 0xffffffff, 0x00060005,
 610	0xf011, 0xffffffff, 0x00090008,
 611	0xf000, 0xffffffff, 0x96e00200,
 612	0x21c2, 0xffffffff, 0x00900100,
 613	0x3109, 0xffffffff, 0x0020003f,
 614	0xe, 0xffffffff, 0x0140001c,
 615	0xf, 0x000f0000, 0x000f0000,
 616	0x88, 0xffffffff, 0xc060000c,
 617	0x89, 0xc0000fff, 0x00000100,
 618	0x82a, 0xffffffff, 0x00000104,
 619	0x1579, 0xff000fff, 0x00000100,
 620	0xc33, 0xc0000fff, 0x00000104,
 621	0x3079, 0x00000001, 0x00000001,
 622	0x3403, 0xff000ff0, 0x00000100,
 623	0x3603, 0xff000ff0, 0x00000100
 624};
 625
 626static const u32 hawaii_golden_spm_registers[] =
 627{
 628	0xc200, 0xe0ffffff, 0xe0000000
 629};
 630
 631static const u32 hawaii_golden_common_registers[] =
 632{
 633	0xc200, 0xffffffff, 0xe0000000,
 634	0xa0d4, 0xffffffff, 0x3a00161a,
 635	0xa0d5, 0xffffffff, 0x0000002e,
 636	0x2684, 0xffffffff, 0x00018208,
 637	0x263e, 0xffffffff, 0x12011003
 638};
 639
 640static const u32 hawaii_golden_registers[] =
 641{
 642	0xcd5, 0x00000333, 0x00000333,
 643	0x2684, 0x00010000, 0x00058208,
 644	0x260c, 0xffffffff, 0x00000000,
 645	0x260d, 0xf00fffff, 0x00000400,
 646	0x260e, 0x0002021c, 0x00020200,
 647	0x31e, 0x00000080, 0x00000000,
 648	0x16ec, 0x000000f0, 0x00000070,
 649	0x16f0, 0xf0311fff, 0x80300000,
 650	0xd43, 0x00810000, 0x408af000,
 651	0x1c0c, 0x31000111, 0x00000011,
 652	0xbd2, 0x73773777, 0x12010001,
 653	0x848, 0x0000007f, 0x0000001b,
 654	0x877, 0x00007fb6, 0x00002191,
 655	0xd8a, 0x0000003f, 0x0000000a,
 656	0xd8b, 0x0000003f, 0x0000000a,
 657	0xab9, 0x00073ffe, 0x000022a2,
 658	0x903, 0x000007ff, 0x00000000,
 659	0x22fc, 0x00002001, 0x00000001,
 660	0x22c9, 0xffffffff, 0x00ffffff,
 661	0xc281, 0x0000ff0f, 0x00000000,
 662	0xa293, 0x07ffffff, 0x06000000,
 663	0xf9e, 0x00000001, 0x00000002,
 664	0x31da, 0x00000008, 0x00000008,
 665	0x31dc, 0x00000f00, 0x00000800,
 666	0x31dd, 0x00000f00, 0x00000800,
 667	0x31e6, 0x00ffffff, 0x00ff7fbf,
 668	0x31e7, 0x00ffffff, 0x00ff7faf,
 669	0x2300, 0x000000ff, 0x00000800,
 670	0x390, 0x00001fff, 0x00001fff,
 671	0x2418, 0x0000007f, 0x00000020,
 672	0x2542, 0x00010000, 0x00010000,
 673	0x2b80, 0x00100000, 0x000ff07c,
 674	0x2b05, 0x000003ff, 0x0000000f,
 675	0x2b04, 0xffffffff, 0x7564fdec,
 676	0x2b03, 0xffffffff, 0x3120b9a8,
 677	0x2b02, 0x20000000, 0x0f9c0000
 678};
 679
 680static const u32 hawaii_mgcg_cgcg_init[] =
 681{
 682	0x3108, 0xffffffff, 0xfffffffd,
 683	0xc200, 0xffffffff, 0xe0000000,
 684	0xf0a8, 0xffffffff, 0x00000100,
 685	0xf082, 0xffffffff, 0x00000100,
 686	0xf0b0, 0xffffffff, 0x00000100,
 687	0xf0b2, 0xffffffff, 0x00000100,
 688	0xf0b1, 0xffffffff, 0x00000100,
 689	0x1579, 0xffffffff, 0x00200100,
 690	0xf0a0, 0xffffffff, 0x00000100,
 691	0xf085, 0xffffffff, 0x06000100,
 692	0xf088, 0xffffffff, 0x00000100,
 693	0xf086, 0xffffffff, 0x06000100,
 694	0xf081, 0xffffffff, 0x00000100,
 695	0xf0b8, 0xffffffff, 0x00000100,
 696	0xf089, 0xffffffff, 0x00000100,
 697	0xf080, 0xffffffff, 0x00000100,
 698	0xf08c, 0xffffffff, 0x00000100,
 699	0xf08d, 0xffffffff, 0x00000100,
 700	0xf094, 0xffffffff, 0x00000100,
 701	0xf095, 0xffffffff, 0x00000100,
 702	0xf096, 0xffffffff, 0x00000100,
 703	0xf097, 0xffffffff, 0x00000100,
 704	0xf098, 0xffffffff, 0x00000100,
 705	0xf09f, 0xffffffff, 0x00000100,
 706	0xf09e, 0xffffffff, 0x00000100,
 707	0xf084, 0xffffffff, 0x06000100,
 708	0xf0a4, 0xffffffff, 0x00000100,
 709	0xf09d, 0xffffffff, 0x00000100,
 710	0xf0ad, 0xffffffff, 0x00000100,
 711	0xf0ac, 0xffffffff, 0x00000100,
 712	0xf09c, 0xffffffff, 0x00000100,
 713	0xc200, 0xffffffff, 0xe0000000,
 714	0xf008, 0xffffffff, 0x00010000,
 715	0xf009, 0xffffffff, 0x00030002,
 716	0xf00a, 0xffffffff, 0x00040007,
 717	0xf00b, 0xffffffff, 0x00060005,
 718	0xf00c, 0xffffffff, 0x00090008,
 719	0xf00d, 0xffffffff, 0x00010000,
 720	0xf00e, 0xffffffff, 0x00030002,
 721	0xf00f, 0xffffffff, 0x00040007,
 722	0xf010, 0xffffffff, 0x00060005,
 723	0xf011, 0xffffffff, 0x00090008,
 724	0xf012, 0xffffffff, 0x00010000,
 725	0xf013, 0xffffffff, 0x00030002,
 726	0xf014, 0xffffffff, 0x00040007,
 727	0xf015, 0xffffffff, 0x00060005,
 728	0xf016, 0xffffffff, 0x00090008,
 729	0xf017, 0xffffffff, 0x00010000,
 730	0xf018, 0xffffffff, 0x00030002,
 731	0xf019, 0xffffffff, 0x00040007,
 732	0xf01a, 0xffffffff, 0x00060005,
 733	0xf01b, 0xffffffff, 0x00090008,
 734	0xf01c, 0xffffffff, 0x00010000,
 735	0xf01d, 0xffffffff, 0x00030002,
 736	0xf01e, 0xffffffff, 0x00040007,
 737	0xf01f, 0xffffffff, 0x00060005,
 738	0xf020, 0xffffffff, 0x00090008,
 739	0xf021, 0xffffffff, 0x00010000,
 740	0xf022, 0xffffffff, 0x00030002,
 741	0xf023, 0xffffffff, 0x00040007,
 742	0xf024, 0xffffffff, 0x00060005,
 743	0xf025, 0xffffffff, 0x00090008,
 744	0xf026, 0xffffffff, 0x00010000,
 745	0xf027, 0xffffffff, 0x00030002,
 746	0xf028, 0xffffffff, 0x00040007,
 747	0xf029, 0xffffffff, 0x00060005,
 748	0xf02a, 0xffffffff, 0x00090008,
 749	0xf02b, 0xffffffff, 0x00010000,
 750	0xf02c, 0xffffffff, 0x00030002,
 751	0xf02d, 0xffffffff, 0x00040007,
 752	0xf02e, 0xffffffff, 0x00060005,
 753	0xf02f, 0xffffffff, 0x00090008,
 754	0xf030, 0xffffffff, 0x00010000,
 755	0xf031, 0xffffffff, 0x00030002,
 756	0xf032, 0xffffffff, 0x00040007,
 757	0xf033, 0xffffffff, 0x00060005,
 758	0xf034, 0xffffffff, 0x00090008,
 759	0xf035, 0xffffffff, 0x00010000,
 760	0xf036, 0xffffffff, 0x00030002,
 761	0xf037, 0xffffffff, 0x00040007,
 762	0xf038, 0xffffffff, 0x00060005,
 763	0xf039, 0xffffffff, 0x00090008,
 764	0xf03a, 0xffffffff, 0x00010000,
 765	0xf03b, 0xffffffff, 0x00030002,
 766	0xf03c, 0xffffffff, 0x00040007,
 767	0xf03d, 0xffffffff, 0x00060005,
 768	0xf03e, 0xffffffff, 0x00090008,
 769	0x30c6, 0xffffffff, 0x00020200,
 770	0xcd4, 0xffffffff, 0x00000200,
 771	0x570, 0xffffffff, 0x00000400,
 772	0x157a, 0xffffffff, 0x00000000,
 773	0xbd4, 0xffffffff, 0x00000902,
 774	0xf000, 0xffffffff, 0x96940200,
 775	0x21c2, 0xffffffff, 0x00900100,
 776	0x3109, 0xffffffff, 0x0020003f,
 777	0xe, 0xffffffff, 0x0140001c,
 778	0xf, 0x000f0000, 0x000f0000,
 779	0x88, 0xffffffff, 0xc060000c,
 780	0x89, 0xc0000fff, 0x00000100,
 781	0x3e4, 0xffffffff, 0x00000100,
 782	0x3e6, 0x00000101, 0x00000000,
 783	0x82a, 0xffffffff, 0x00000104,
 784	0x1579, 0xff000fff, 0x00000100,
 785	0xc33, 0xc0000fff, 0x00000104,
 786	0x3079, 0x00000001, 0x00000001,
 787	0x3403, 0xff000ff0, 0x00000100,
 788	0x3603, 0xff000ff0, 0x00000100
 789};
 790
 791static const u32 godavari_golden_registers[] =
 792{
 793	0x1579, 0xff607fff, 0xfc000100,
 794	0x1bb6, 0x00010101, 0x00010000,
 795	0x260c, 0xffffffff, 0x00000000,
 796	0x260c0, 0xf00fffff, 0x00000400,
 797	0x184c, 0xffffffff, 0x00010000,
 798	0x16ec, 0x000000f0, 0x00000070,
 799	0x16f0, 0xf0311fff, 0x80300000,
 800	0x263e, 0x73773777, 0x12010001,
 801	0x263f, 0xffffffff, 0x00000010,
 802	0x200c, 0x00001f0f, 0x0000100a,
 803	0xbd2, 0x73773777, 0x12010001,
 804	0x902, 0x000fffff, 0x000c007f,
 805	0x2285, 0xf000003f, 0x00000007,
 806	0x22c9, 0xffffffff, 0x00ff0fff,
 807	0xc281, 0x0000ff0f, 0x00000000,
 808	0xa293, 0x07ffffff, 0x06000000,
 809	0x136, 0x00000fff, 0x00000100,
 810	0x3405, 0x00010000, 0x00810001,
 811	0x3605, 0x00010000, 0x00810001,
 812	0xf9e, 0x00000001, 0x00000002,
 813	0x31da, 0x00000008, 0x00000008,
 814	0x31dc, 0x00000f00, 0x00000800,
 815	0x31dd, 0x00000f00, 0x00000800,
 816	0x31e6, 0x00ffffff, 0x00ff7fbf,
 817	0x31e7, 0x00ffffff, 0x00ff7faf,
 818	0x2300, 0x000000ff, 0x00000001,
 819	0x853e, 0x01ff01ff, 0x00000002,
 820	0x8526, 0x007ff800, 0x00200000,
 821	0x8057, 0xffffffff, 0x00000f40,
 822	0x2231, 0x001f3ae3, 0x00000082,
 823	0x2235, 0x0000001f, 0x00000010,
 824	0xc24d, 0xffffffff, 0x00000000
 825};
 826
 827static void cik_init_golden_registers(struct amdgpu_device *adev)
 828{
 829	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 830	mutex_lock(&adev->grbm_idx_mutex);
 831
 832	switch (adev->asic_type) {
 833	case CHIP_BONAIRE:
 834		amdgpu_device_program_register_sequence(adev,
 835							bonaire_mgcg_cgcg_init,
 836							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 837		amdgpu_device_program_register_sequence(adev,
 838							bonaire_golden_registers,
 839							ARRAY_SIZE(bonaire_golden_registers));
 840		amdgpu_device_program_register_sequence(adev,
 841							bonaire_golden_common_registers,
 842							ARRAY_SIZE(bonaire_golden_common_registers));
 843		amdgpu_device_program_register_sequence(adev,
 844							bonaire_golden_spm_registers,
 845							ARRAY_SIZE(bonaire_golden_spm_registers));
 846		break;
 847	case CHIP_KABINI:
 848		amdgpu_device_program_register_sequence(adev,
 849							kalindi_mgcg_cgcg_init,
 850							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 851		amdgpu_device_program_register_sequence(adev,
 852							kalindi_golden_registers,
 853							ARRAY_SIZE(kalindi_golden_registers));
 854		amdgpu_device_program_register_sequence(adev,
 855							kalindi_golden_common_registers,
 856							ARRAY_SIZE(kalindi_golden_common_registers));
 857		amdgpu_device_program_register_sequence(adev,
 858							kalindi_golden_spm_registers,
 859							ARRAY_SIZE(kalindi_golden_spm_registers));
 860		break;
 861	case CHIP_MULLINS:
 862		amdgpu_device_program_register_sequence(adev,
 863							kalindi_mgcg_cgcg_init,
 864							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 865		amdgpu_device_program_register_sequence(adev,
 866							godavari_golden_registers,
 867							ARRAY_SIZE(godavari_golden_registers));
 868		amdgpu_device_program_register_sequence(adev,
 869							kalindi_golden_common_registers,
 870							ARRAY_SIZE(kalindi_golden_common_registers));
 871		amdgpu_device_program_register_sequence(adev,
 872							kalindi_golden_spm_registers,
 873							ARRAY_SIZE(kalindi_golden_spm_registers));
 874		break;
 875	case CHIP_KAVERI:
 876		amdgpu_device_program_register_sequence(adev,
 877							spectre_mgcg_cgcg_init,
 878							ARRAY_SIZE(spectre_mgcg_cgcg_init));
 879		amdgpu_device_program_register_sequence(adev,
 880							spectre_golden_registers,
 881							ARRAY_SIZE(spectre_golden_registers));
 882		amdgpu_device_program_register_sequence(adev,
 883							spectre_golden_common_registers,
 884							ARRAY_SIZE(spectre_golden_common_registers));
 885		amdgpu_device_program_register_sequence(adev,
 886							spectre_golden_spm_registers,
 887							ARRAY_SIZE(spectre_golden_spm_registers));
 888		break;
 889	case CHIP_HAWAII:
 890		amdgpu_device_program_register_sequence(adev,
 891							hawaii_mgcg_cgcg_init,
 892							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 893		amdgpu_device_program_register_sequence(adev,
 894							hawaii_golden_registers,
 895							ARRAY_SIZE(hawaii_golden_registers));
 896		amdgpu_device_program_register_sequence(adev,
 897							hawaii_golden_common_registers,
 898							ARRAY_SIZE(hawaii_golden_common_registers));
 899		amdgpu_device_program_register_sequence(adev,
 900							hawaii_golden_spm_registers,
 901							ARRAY_SIZE(hawaii_golden_spm_registers));
 902		break;
 903	default:
 904		break;
 905	}
 906	mutex_unlock(&adev->grbm_idx_mutex);
 907}
 908
 909/**
 910 * cik_get_xclk - get the xclk
 911 *
 912 * @adev: amdgpu_device pointer
 913 *
 914 * Returns the reference clock used by the gfx engine
 915 * (CIK).
 916 */
 917static u32 cik_get_xclk(struct amdgpu_device *adev)
 918{
 919	u32 reference_clock = adev->clock.spll.reference_freq;
 920
 921	if (adev->flags & AMD_IS_APU) {
 922		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 923			return reference_clock / 2;
 924	} else {
 925		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 926			return reference_clock / 4;
 927	}
 928	return reference_clock;
 929}
 930
 931/**
 932 * cik_srbm_select - select specific register instances
 933 *
 934 * @adev: amdgpu_device pointer
 935 * @me: selected ME (micro engine)
 936 * @pipe: pipe
 937 * @queue: queue
 938 * @vmid: VMID
 939 *
 940 * Switches the currently active registers instances.  Some
 941 * registers are instanced per VMID, others are instanced per
 942 * me/pipe/queue combination.
 943 */
 944void cik_srbm_select(struct amdgpu_device *adev,
 945		     u32 me, u32 pipe, u32 queue, u32 vmid)
 946{
 947	u32 srbm_gfx_cntl =
 948		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 949		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 950		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 951		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 952	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 953}
 954
 955static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 956{
 957	uint32_t tmp;
 958
 959	tmp = RREG32(mmCONFIG_CNTL);
 960	if (!state)
 961		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 962	else
 963		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 964	WREG32(mmCONFIG_CNTL, tmp);
 965}
 966
 967static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 968{
 969	u32 bus_cntl;
 970	u32 d1vga_control = 0;
 971	u32 d2vga_control = 0;
 972	u32 vga_render_control = 0;
 973	u32 rom_cntl;
 974	bool r;
 975
 976	bus_cntl = RREG32(mmBUS_CNTL);
 977	if (adev->mode_info.num_crtc) {
 978		d1vga_control = RREG32(mmD1VGA_CONTROL);
 979		d2vga_control = RREG32(mmD2VGA_CONTROL);
 980		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 981	}
 982	rom_cntl = RREG32_SMC(ixROM_CNTL);
 983
 984	/* enable the rom */
 985	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 986	if (adev->mode_info.num_crtc) {
 987		/* Disable VGA mode */
 988		WREG32(mmD1VGA_CONTROL,
 989		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 990					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 991		WREG32(mmD2VGA_CONTROL,
 992		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 993					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 994		WREG32(mmVGA_RENDER_CONTROL,
 995		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 996	}
 997	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 998
 999	r = amdgpu_read_bios(adev);
1000
1001	/* restore regs */
1002	WREG32(mmBUS_CNTL, bus_cntl);
1003	if (adev->mode_info.num_crtc) {
1004		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007	}
1008	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009	return r;
1010}
1011
1012static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013				   u8 *bios, u32 length_bytes)
1014{
1015	u32 *dw_ptr;
1016	unsigned long flags;
1017	u32 i, length_dw;
1018
1019	if (bios == NULL)
1020		return false;
1021	if (length_bytes == 0)
1022		return false;
1023	/* APU vbios image is part of sbios image */
1024	if (adev->flags & AMD_IS_APU)
1025		return false;
1026
1027	dw_ptr = (u32 *)bios;
1028	length_dw = ALIGN(length_bytes, 4) / 4;
1029	/* take the smc lock since we are using the smc index */
1030	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031	/* set rom index to 0 */
1032	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033	WREG32(mmSMC_IND_DATA_0, 0);
1034	/* set index to data for continous read */
1035	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036	for (i = 0; i < length_dw; i++)
1037		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040	return true;
1041}
1042
1043static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044	{mmGRBM_STATUS},
1045	{mmGRBM_STATUS2},
1046	{mmGRBM_STATUS_SE0},
1047	{mmGRBM_STATUS_SE1},
1048	{mmGRBM_STATUS_SE2},
1049	{mmGRBM_STATUS_SE3},
1050	{mmSRBM_STATUS},
1051	{mmSRBM_STATUS2},
1052	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054	{mmCP_STAT},
1055	{mmCP_STALLED_STAT1},
1056	{mmCP_STALLED_STAT2},
1057	{mmCP_STALLED_STAT3},
1058	{mmCP_CPF_BUSY_STAT},
1059	{mmCP_CPF_STALLED_STAT1},
1060	{mmCP_CPF_STATUS},
1061	{mmCP_CPC_BUSY_STAT},
1062	{mmCP_CPC_STALLED_STAT1},
1063	{mmCP_CPC_STATUS},
1064	{mmGB_ADDR_CONFIG},
1065	{mmMC_ARB_RAMCFG},
1066	{mmGB_TILE_MODE0},
1067	{mmGB_TILE_MODE1},
1068	{mmGB_TILE_MODE2},
1069	{mmGB_TILE_MODE3},
1070	{mmGB_TILE_MODE4},
1071	{mmGB_TILE_MODE5},
1072	{mmGB_TILE_MODE6},
1073	{mmGB_TILE_MODE7},
1074	{mmGB_TILE_MODE8},
1075	{mmGB_TILE_MODE9},
1076	{mmGB_TILE_MODE10},
1077	{mmGB_TILE_MODE11},
1078	{mmGB_TILE_MODE12},
1079	{mmGB_TILE_MODE13},
1080	{mmGB_TILE_MODE14},
1081	{mmGB_TILE_MODE15},
1082	{mmGB_TILE_MODE16},
1083	{mmGB_TILE_MODE17},
1084	{mmGB_TILE_MODE18},
1085	{mmGB_TILE_MODE19},
1086	{mmGB_TILE_MODE20},
1087	{mmGB_TILE_MODE21},
1088	{mmGB_TILE_MODE22},
1089	{mmGB_TILE_MODE23},
1090	{mmGB_TILE_MODE24},
1091	{mmGB_TILE_MODE25},
1092	{mmGB_TILE_MODE26},
1093	{mmGB_TILE_MODE27},
1094	{mmGB_TILE_MODE28},
1095	{mmGB_TILE_MODE29},
1096	{mmGB_TILE_MODE30},
1097	{mmGB_TILE_MODE31},
1098	{mmGB_MACROTILE_MODE0},
1099	{mmGB_MACROTILE_MODE1},
1100	{mmGB_MACROTILE_MODE2},
1101	{mmGB_MACROTILE_MODE3},
1102	{mmGB_MACROTILE_MODE4},
1103	{mmGB_MACROTILE_MODE5},
1104	{mmGB_MACROTILE_MODE6},
1105	{mmGB_MACROTILE_MODE7},
1106	{mmGB_MACROTILE_MODE8},
1107	{mmGB_MACROTILE_MODE9},
1108	{mmGB_MACROTILE_MODE10},
1109	{mmGB_MACROTILE_MODE11},
1110	{mmGB_MACROTILE_MODE12},
1111	{mmGB_MACROTILE_MODE13},
1112	{mmGB_MACROTILE_MODE14},
1113	{mmGB_MACROTILE_MODE15},
1114	{mmCC_RB_BACKEND_DISABLE, true},
1115	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116	{mmGB_BACKEND_MAP, false},
1117	{mmPA_SC_RASTER_CONFIG, true},
1118	{mmPA_SC_RASTER_CONFIG_1, true},
1119};
1120
1121
1122static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123				       bool indexed, u32 se_num,
1124				       u32 sh_num, u32 reg_offset)
1125{
1126	if (indexed) {
1127		uint32_t val;
1128		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131		switch (reg_offset) {
1132		case mmCC_RB_BACKEND_DISABLE:
1133			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134		case mmGC_USER_RB_BACKEND_DISABLE:
1135			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136		case mmPA_SC_RASTER_CONFIG:
1137			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138		case mmPA_SC_RASTER_CONFIG_1:
1139			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140		}
1141
1142		mutex_lock(&adev->grbm_idx_mutex);
1143		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145
1146		val = RREG32(reg_offset);
1147
1148		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150		mutex_unlock(&adev->grbm_idx_mutex);
1151		return val;
1152	} else {
1153		unsigned idx;
1154
1155		switch (reg_offset) {
1156		case mmGB_ADDR_CONFIG:
1157			return adev->gfx.config.gb_addr_config;
1158		case mmMC_ARB_RAMCFG:
1159			return adev->gfx.config.mc_arb_ramcfg;
1160		case mmGB_TILE_MODE0:
1161		case mmGB_TILE_MODE1:
1162		case mmGB_TILE_MODE2:
1163		case mmGB_TILE_MODE3:
1164		case mmGB_TILE_MODE4:
1165		case mmGB_TILE_MODE5:
1166		case mmGB_TILE_MODE6:
1167		case mmGB_TILE_MODE7:
1168		case mmGB_TILE_MODE8:
1169		case mmGB_TILE_MODE9:
1170		case mmGB_TILE_MODE10:
1171		case mmGB_TILE_MODE11:
1172		case mmGB_TILE_MODE12:
1173		case mmGB_TILE_MODE13:
1174		case mmGB_TILE_MODE14:
1175		case mmGB_TILE_MODE15:
1176		case mmGB_TILE_MODE16:
1177		case mmGB_TILE_MODE17:
1178		case mmGB_TILE_MODE18:
1179		case mmGB_TILE_MODE19:
1180		case mmGB_TILE_MODE20:
1181		case mmGB_TILE_MODE21:
1182		case mmGB_TILE_MODE22:
1183		case mmGB_TILE_MODE23:
1184		case mmGB_TILE_MODE24:
1185		case mmGB_TILE_MODE25:
1186		case mmGB_TILE_MODE26:
1187		case mmGB_TILE_MODE27:
1188		case mmGB_TILE_MODE28:
1189		case mmGB_TILE_MODE29:
1190		case mmGB_TILE_MODE30:
1191		case mmGB_TILE_MODE31:
1192			idx = (reg_offset - mmGB_TILE_MODE0);
1193			return adev->gfx.config.tile_mode_array[idx];
1194		case mmGB_MACROTILE_MODE0:
1195		case mmGB_MACROTILE_MODE1:
1196		case mmGB_MACROTILE_MODE2:
1197		case mmGB_MACROTILE_MODE3:
1198		case mmGB_MACROTILE_MODE4:
1199		case mmGB_MACROTILE_MODE5:
1200		case mmGB_MACROTILE_MODE6:
1201		case mmGB_MACROTILE_MODE7:
1202		case mmGB_MACROTILE_MODE8:
1203		case mmGB_MACROTILE_MODE9:
1204		case mmGB_MACROTILE_MODE10:
1205		case mmGB_MACROTILE_MODE11:
1206		case mmGB_MACROTILE_MODE12:
1207		case mmGB_MACROTILE_MODE13:
1208		case mmGB_MACROTILE_MODE14:
1209		case mmGB_MACROTILE_MODE15:
1210			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211			return adev->gfx.config.macrotile_mode_array[idx];
1212		default:
1213			return RREG32(reg_offset);
1214		}
1215	}
1216}
1217
1218static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219			     u32 sh_num, u32 reg_offset, u32 *value)
1220{
1221	uint32_t i;
1222
1223	*value = 0;
1224	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228			continue;
1229
1230		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231						reg_offset);
 
 
 
1232		return 0;
1233	}
1234	return -EINVAL;
1235}
1236
1237struct kv_reset_save_regs {
1238	u32 gmcon_reng_execute;
1239	u32 gmcon_misc;
1240	u32 gmcon_misc3;
1241};
1242
1243static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244				   struct kv_reset_save_regs *save)
1245{
1246	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255}
1256
1257static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258				      struct kv_reset_save_regs *save)
1259{
1260	int i;
1261
1262	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265	for (i = 0; i < 5; i++)
1266		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271	for (i = 0; i < 5; i++)
1272		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277	for (i = 0; i < 5; i++)
1278		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283	for (i = 0; i < 5; i++)
1284		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289	for (i = 0; i < 5; i++)
1290		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295	for (i = 0; i < 5; i++)
1296		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301	for (i = 0; i < 5; i++)
1302		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307	for (i = 0; i < 5; i++)
1308		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313	for (i = 0; i < 5; i++)
1314		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319	for (i = 0; i < 5; i++)
1320		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328}
1329
1330/**
1331 * cik_asic_pci_config_reset - soft reset GPU
1332 *
1333 * @adev: amdgpu_device pointer
1334 *
1335 * Use PCI Config method to reset the GPU.
1336 *
1337 * Returns 0 for success.
1338 */
1339static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340{
1341	struct kv_reset_save_regs kv_save = { 0 };
1342	u32 i;
1343	int r = -EINVAL;
1344
1345	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347	if (adev->flags & AMD_IS_APU)
1348		kv_save_regs_for_reset(adev, &kv_save);
1349
1350	/* disable BM */
1351	pci_clear_master(adev->pdev);
1352	/* reset */
1353	amdgpu_device_pci_config_reset(adev);
1354
1355	udelay(100);
1356
1357	/* wait for asic to come out of reset */
1358	for (i = 0; i < adev->usec_timeout; i++) {
1359		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360			/* enable BM */
1361			pci_set_master(adev->pdev);
1362			adev->has_hw_reset = true;
1363			r = 0;
1364			break;
1365		}
1366		udelay(1);
1367	}
1368
1369	/* does asic init need to be run first??? */
1370	if (adev->flags & AMD_IS_APU)
1371		kv_restore_regs_for_reset(adev, &kv_save);
1372
1373	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375	return r;
1376}
1377
1378static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379{
1380	switch (adev->asic_type) {
1381	case CHIP_BONAIRE:
1382	case CHIP_HAWAII:
1383		return amdgpu_dpm_is_baco_supported(adev);
1384	default:
1385		return false;
1386	}
1387}
1388
1389static enum amd_reset_method
1390cik_asic_reset_method(struct amdgpu_device *adev)
1391{
1392	bool baco_reset;
1393
1394	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396		return amdgpu_reset_method;
1397
1398	if (amdgpu_reset_method != -1)
1399		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400				  amdgpu_reset_method);
1401
1402	switch (adev->asic_type) {
1403	case CHIP_BONAIRE:
1404	case CHIP_HAWAII:
1405		baco_reset = cik_asic_supports_baco(adev);
1406		break;
1407	default:
1408		baco_reset = false;
1409		break;
1410	}
1411
1412	if (baco_reset)
1413		return AMD_RESET_METHOD_BACO;
1414	else
1415		return AMD_RESET_METHOD_LEGACY;
1416}
1417
1418/**
1419 * cik_asic_reset - soft reset GPU
1420 *
1421 * @adev: amdgpu_device pointer
1422 *
1423 * Look up which blocks are hung and attempt
1424 * to reset them.
1425 * Returns 0 for success.
1426 */
1427static int cik_asic_reset(struct amdgpu_device *adev)
1428{
1429	int r;
1430
1431	/* APUs don't have full asic reset */
1432	if (adev->flags & AMD_IS_APU)
1433		return 0;
1434
1435	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436		dev_info(adev->dev, "BACO reset\n");
1437		r = amdgpu_dpm_baco_reset(adev);
1438	} else {
1439		dev_info(adev->dev, "PCI CONFIG reset\n");
1440		r = cik_asic_pci_config_reset(adev);
1441	}
1442
1443	return r;
1444}
1445
1446static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447{
1448	return RREG32(mmCONFIG_MEMSIZE);
1449}
1450
1451static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452			      u32 cntl_reg, u32 status_reg)
1453{
1454	int r, i;
1455	struct atom_clock_dividers dividers;
1456	uint32_t tmp;
1457
1458	r = amdgpu_atombios_get_clock_dividers(adev,
1459					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460					       clock, false, &dividers);
1461	if (r)
1462		return r;
1463
1464	tmp = RREG32_SMC(cntl_reg);
1465	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467	tmp |= dividers.post_divider;
1468	WREG32_SMC(cntl_reg, tmp);
1469
1470	for (i = 0; i < 100; i++) {
1471		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472			break;
1473		mdelay(10);
1474	}
1475	if (i == 100)
1476		return -ETIMEDOUT;
1477
1478	return 0;
1479}
1480
1481static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482{
1483	int r = 0;
1484
1485	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486	if (r)
1487		return r;
1488
1489	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490	return r;
1491}
1492
1493static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494{
1495	int r, i;
1496	struct atom_clock_dividers dividers;
1497	u32 tmp;
1498
1499	r = amdgpu_atombios_get_clock_dividers(adev,
1500					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501					       ecclk, false, &dividers);
1502	if (r)
1503		return r;
1504
1505	for (i = 0; i < 100; i++) {
1506		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507			break;
1508		mdelay(10);
1509	}
1510	if (i == 100)
1511		return -ETIMEDOUT;
1512
1513	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516	tmp |= dividers.post_divider;
1517	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518
1519	for (i = 0; i < 100; i++) {
1520		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521			break;
1522		mdelay(10);
1523	}
1524	if (i == 100)
1525		return -ETIMEDOUT;
1526
1527	return 0;
1528}
1529
1530static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531{
1532	struct pci_dev *root = adev->pdev->bus->self;
 
1533	u32 speed_cntl, current_data_rate;
1534	int i;
1535	u16 tmp16;
1536
1537	if (pci_is_root_bus(adev->pdev->bus))
1538		return;
1539
1540	if (amdgpu_pcie_gen2 == 0)
1541		return;
1542
1543	if (adev->flags & AMD_IS_APU)
1544		return;
1545
1546	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548		return;
1549
1550	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554		if (current_data_rate == 2) {
1555			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556			return;
1557		}
1558		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560		if (current_data_rate == 1) {
1561			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562			return;
1563		}
1564		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565	}
1566
1567	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
 
 
 
 
 
1568		return;
1569
1570	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571		/* re-try equalization if gen3 is not already enabled */
1572		if (current_data_rate != 2) {
1573			u16 bridge_cfg, gpu_cfg;
1574			u16 bridge_cfg2, gpu_cfg2;
1575			u32 max_lw, current_lw, tmp;
1576
1577			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
1579
1580			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585
1586			if (current_lw < max_lw) {
1587				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591					tmp |= (max_lw <<
1592						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597				}
1598			}
1599
1600			for (i = 0; i < 10; i++) {
1601				/* check status */
1602				pcie_capability_read_word(adev->pdev,
1603							  PCI_EXP_DEVSTA,
1604							  &tmp16);
1605				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606					break;
1607
1608				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609							  &bridge_cfg);
1610				pcie_capability_read_word(adev->pdev,
1611							  PCI_EXP_LNKCTL,
1612							  &gpu_cfg);
1613
1614				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615							  &bridge_cfg2);
1616				pcie_capability_read_word(adev->pdev,
1617							  PCI_EXP_LNKCTL2,
1618							  &gpu_cfg2);
1619
1620				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623
1624				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627
1628				msleep(100);
1629
1630				/* linkctl */
1631				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632								   PCI_EXP_LNKCTL_HAWD,
1633								   bridge_cfg &
1634								   PCI_EXP_LNKCTL_HAWD);
1635				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636								   PCI_EXP_LNKCTL_HAWD,
1637								   gpu_cfg &
1638								   PCI_EXP_LNKCTL_HAWD);
 
1639
1640				/* linkctl2 */
1641				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1642							  &tmp16);
1643				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1644					   PCI_EXP_LNKCTL2_TX_MARGIN);
1645				tmp16 |= (bridge_cfg2 &
1646					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1647					   PCI_EXP_LNKCTL2_TX_MARGIN));
1648				pcie_capability_write_word(root,
1649							   PCI_EXP_LNKCTL2,
1650							   tmp16);
1651
1652				pcie_capability_read_word(adev->pdev,
1653							  PCI_EXP_LNKCTL2,
1654							  &tmp16);
1655				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656					   PCI_EXP_LNKCTL2_TX_MARGIN);
1657				tmp16 |= (gpu_cfg2 &
1658					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1659					   PCI_EXP_LNKCTL2_TX_MARGIN));
1660				pcie_capability_write_word(adev->pdev,
1661							   PCI_EXP_LNKCTL2,
1662							   tmp16);
1663
1664				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1665				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1666				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1667			}
1668		}
1669	}
1670
1671	/* set the link speed */
1672	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1673		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1674	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1675	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1676
1677	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1678	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1679
1680	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1681		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1682	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1683		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1684	else
1685		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1686	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1687
1688	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1689	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1690	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1691
1692	for (i = 0; i < adev->usec_timeout; i++) {
1693		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1694		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1695			break;
1696		udelay(1);
1697	}
1698}
1699
1700static void cik_program_aspm(struct amdgpu_device *adev)
1701{
1702	u32 data, orig;
1703	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1704	bool disable_clkreq = false;
1705
1706	if (!amdgpu_device_should_use_aspm(adev))
1707		return;
1708
1709	if (pci_is_root_bus(adev->pdev->bus))
1710		return;
1711
 
 
 
 
1712	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1713	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1714	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1715		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1716	if (orig != data)
1717		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1718
1719	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1720	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1721	if (orig != data)
1722		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1723
1724	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1725	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1726	if (orig != data)
1727		WREG32_PCIE(ixPCIE_P_CNTL, data);
1728
1729	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1730	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1731		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1732	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1733	if (!disable_l0s)
1734		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1735
1736	if (!disable_l1) {
1737		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1738		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1739		if (orig != data)
1740			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1741
1742		if (!disable_plloff_in_l1) {
1743			bool clk_req_support;
1744
1745			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1746			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1747				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1748			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1749				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1750			if (orig != data)
1751				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1752
1753			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1754			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1755				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1756			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1757				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1758			if (orig != data)
1759				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1760
1761			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1762			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1763				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1764			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1765				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1766			if (orig != data)
1767				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1768
1769			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1770			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1771				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1772			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1773				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1774			if (orig != data)
1775				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1776
1777			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1778			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1779			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1780			if (orig != data)
1781				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1782
1783			if (!disable_clkreq) {
1784				struct pci_dev *root = adev->pdev->bus->self;
1785				u32 lnkcap;
1786
1787				clk_req_support = false;
1788				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1789				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1790					clk_req_support = true;
1791			} else {
1792				clk_req_support = false;
1793			}
1794
1795			if (clk_req_support) {
1796				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1797				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1798					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1799				if (orig != data)
1800					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1801
1802				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1803				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1804					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1805				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1806					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1807				if (orig != data)
1808					WREG32_SMC(ixTHM_CLK_CNTL, data);
1809
1810				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1811				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1812					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1813				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1814					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1815				if (orig != data)
1816					WREG32_SMC(ixMISC_CLK_CTRL, data);
1817
1818				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1819				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1820				if (orig != data)
1821					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1822
1823				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1824				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1825				if (orig != data)
1826					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1827
1828				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1829				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1830				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1831				if (orig != data)
1832					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1833			}
1834		}
1835	} else {
1836		if (orig != data)
1837			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1838	}
1839
1840	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1841	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1842		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1843		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1844	if (orig != data)
1845		WREG32_PCIE(ixPCIE_CNTL2, data);
1846
1847	if (!disable_l0s) {
1848		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1849		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1850				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1851			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1852			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1853			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1854				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1855				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1856				if (orig != data)
1857					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1858			}
1859		}
1860	}
1861}
1862
1863static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1864{
1865	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1866		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1867}
1868
1869static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1870{
1871	if (!ring || !ring->funcs->emit_wreg) {
1872		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1873		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1874	} else {
1875		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1876	}
1877}
1878
1879static void cik_invalidate_hdp(struct amdgpu_device *adev,
1880			       struct amdgpu_ring *ring)
1881{
1882	if (!ring || !ring->funcs->emit_wreg) {
1883		WREG32(mmHDP_DEBUG0, 1);
1884		RREG32(mmHDP_DEBUG0);
1885	} else {
1886		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1887	}
1888}
1889
1890static bool cik_need_full_reset(struct amdgpu_device *adev)
1891{
1892	/* change this when we support soft reset */
1893	return true;
1894}
1895
1896static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1897			       uint64_t *count1)
1898{
1899	uint32_t perfctr = 0;
1900	uint64_t cnt0_of, cnt1_of;
1901	int tmp;
1902
1903	/* This reports 0 on APUs, so return to avoid writing/reading registers
1904	 * that may or may not be different from their GPU counterparts
1905	 */
1906	if (adev->flags & AMD_IS_APU)
1907		return;
1908
1909	/* Set the 2 events that we wish to watch, defined above */
1910	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1911	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1912	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1913
1914	/* Write to enable desired perf counters */
1915	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1916	/* Zero out and enable the perf counters
1917	 * Write 0x5:
1918	 * Bit 0 = Start all counters(1)
1919	 * Bit 2 = Global counter reset enable(1)
1920	 */
1921	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1922
1923	msleep(1000);
1924
1925	/* Load the shadow and disable the perf counters
1926	 * Write 0x2:
1927	 * Bit 0 = Stop counters(0)
1928	 * Bit 1 = Load the shadow counters(1)
1929	 */
1930	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1931
1932	/* Read register values to get any >32bit overflow */
1933	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1934	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1935	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1936
1937	/* Get the values and add the overflow */
1938	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1939	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1940}
1941
1942static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1943{
1944	u32 clock_cntl, pc;
1945
1946	if (adev->flags & AMD_IS_APU)
1947		return false;
1948
1949	/* check if the SMC is already running */
1950	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1951	pc = RREG32_SMC(ixSMC_PC_C);
1952	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1953	    (0x20100 <= pc))
1954		return true;
1955
1956	return false;
1957}
1958
1959static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1960{
1961	uint64_t nak_r, nak_g;
1962
1963	/* Get the number of NAKs received and generated */
1964	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1965	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1966
1967	/* Add the total number of NAKs, i.e the number of replays */
1968	return (nak_r + nak_g);
1969}
1970
1971static void cik_pre_asic_init(struct amdgpu_device *adev)
1972{
 
 
1973}
1974
1975static const struct amdgpu_asic_funcs cik_asic_funcs =
1976{
1977	.read_disabled_bios = &cik_read_disabled_bios,
1978	.read_bios_from_rom = &cik_read_bios_from_rom,
 
1979	.read_register = &cik_read_register,
1980	.reset = &cik_asic_reset,
1981	.reset_method = &cik_asic_reset_method,
1982	.set_vga_state = &cik_vga_set_state,
1983	.get_xclk = &cik_get_xclk,
1984	.set_uvd_clocks = &cik_set_uvd_clocks,
1985	.set_vce_clocks = &cik_set_vce_clocks,
1986	.get_config_memsize = &cik_get_config_memsize,
1987	.flush_hdp = &cik_flush_hdp,
1988	.invalidate_hdp = &cik_invalidate_hdp,
1989	.need_full_reset = &cik_need_full_reset,
1990	.init_doorbell_index = &legacy_doorbell_index_init,
1991	.get_pcie_usage = &cik_get_pcie_usage,
1992	.need_reset_on_init = &cik_need_reset_on_init,
1993	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1994	.supports_baco = &cik_asic_supports_baco,
1995	.pre_asic_init = &cik_pre_asic_init,
1996	.query_video_codecs = &cik_query_video_codecs,
1997};
1998
1999static int cik_common_early_init(void *handle)
2000{
2001	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2002
2003	adev->smc_rreg = &cik_smc_rreg;
2004	adev->smc_wreg = &cik_smc_wreg;
2005	adev->pcie_rreg = &cik_pcie_rreg;
2006	adev->pcie_wreg = &cik_pcie_wreg;
2007	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2008	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2009	adev->didt_rreg = &cik_didt_rreg;
2010	adev->didt_wreg = &cik_didt_wreg;
2011
2012	adev->asic_funcs = &cik_asic_funcs;
2013
2014	adev->rev_id = cik_get_rev_id(adev);
2015	adev->external_rev_id = 0xFF;
2016	switch (adev->asic_type) {
2017	case CHIP_BONAIRE:
2018		adev->cg_flags =
2019			AMD_CG_SUPPORT_GFX_MGCG |
2020			AMD_CG_SUPPORT_GFX_MGLS |
2021			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2022			AMD_CG_SUPPORT_GFX_CGLS |
2023			AMD_CG_SUPPORT_GFX_CGTS |
2024			AMD_CG_SUPPORT_GFX_CGTS_LS |
2025			AMD_CG_SUPPORT_GFX_CP_LS |
2026			AMD_CG_SUPPORT_MC_LS |
2027			AMD_CG_SUPPORT_MC_MGCG |
2028			AMD_CG_SUPPORT_SDMA_MGCG |
2029			AMD_CG_SUPPORT_SDMA_LS |
2030			AMD_CG_SUPPORT_BIF_LS |
2031			AMD_CG_SUPPORT_VCE_MGCG |
2032			AMD_CG_SUPPORT_UVD_MGCG |
2033			AMD_CG_SUPPORT_HDP_LS |
2034			AMD_CG_SUPPORT_HDP_MGCG;
2035		adev->pg_flags = 0;
2036		adev->external_rev_id = adev->rev_id + 0x14;
2037		break;
2038	case CHIP_HAWAII:
2039		adev->cg_flags =
2040			AMD_CG_SUPPORT_GFX_MGCG |
2041			AMD_CG_SUPPORT_GFX_MGLS |
2042			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2043			AMD_CG_SUPPORT_GFX_CGLS |
2044			AMD_CG_SUPPORT_GFX_CGTS |
2045			AMD_CG_SUPPORT_GFX_CP_LS |
2046			AMD_CG_SUPPORT_MC_LS |
2047			AMD_CG_SUPPORT_MC_MGCG |
2048			AMD_CG_SUPPORT_SDMA_MGCG |
2049			AMD_CG_SUPPORT_SDMA_LS |
2050			AMD_CG_SUPPORT_BIF_LS |
2051			AMD_CG_SUPPORT_VCE_MGCG |
2052			AMD_CG_SUPPORT_UVD_MGCG |
2053			AMD_CG_SUPPORT_HDP_LS |
2054			AMD_CG_SUPPORT_HDP_MGCG;
2055		adev->pg_flags = 0;
2056		adev->external_rev_id = 0x28;
2057		break;
2058	case CHIP_KAVERI:
2059		adev->cg_flags =
2060			AMD_CG_SUPPORT_GFX_MGCG |
2061			AMD_CG_SUPPORT_GFX_MGLS |
2062			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2063			AMD_CG_SUPPORT_GFX_CGLS |
2064			AMD_CG_SUPPORT_GFX_CGTS |
2065			AMD_CG_SUPPORT_GFX_CGTS_LS |
2066			AMD_CG_SUPPORT_GFX_CP_LS |
2067			AMD_CG_SUPPORT_SDMA_MGCG |
2068			AMD_CG_SUPPORT_SDMA_LS |
2069			AMD_CG_SUPPORT_BIF_LS |
2070			AMD_CG_SUPPORT_VCE_MGCG |
2071			AMD_CG_SUPPORT_UVD_MGCG |
2072			AMD_CG_SUPPORT_HDP_LS |
2073			AMD_CG_SUPPORT_HDP_MGCG;
2074		adev->pg_flags =
2075			/*AMD_PG_SUPPORT_GFX_PG |
2076			  AMD_PG_SUPPORT_GFX_SMG |
2077			  AMD_PG_SUPPORT_GFX_DMG |*/
2078			AMD_PG_SUPPORT_UVD |
2079			AMD_PG_SUPPORT_VCE |
2080			/*  AMD_PG_SUPPORT_CP |
2081			  AMD_PG_SUPPORT_GDS |
2082			  AMD_PG_SUPPORT_RLC_SMU_HS |
2083			  AMD_PG_SUPPORT_ACP |
2084			  AMD_PG_SUPPORT_SAMU |*/
2085			0;
2086		if (adev->pdev->device == 0x1312 ||
2087			adev->pdev->device == 0x1316 ||
2088			adev->pdev->device == 0x1317)
2089			adev->external_rev_id = 0x41;
2090		else
2091			adev->external_rev_id = 0x1;
2092		break;
2093	case CHIP_KABINI:
2094	case CHIP_MULLINS:
2095		adev->cg_flags =
2096			AMD_CG_SUPPORT_GFX_MGCG |
2097			AMD_CG_SUPPORT_GFX_MGLS |
2098			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2099			AMD_CG_SUPPORT_GFX_CGLS |
2100			AMD_CG_SUPPORT_GFX_CGTS |
2101			AMD_CG_SUPPORT_GFX_CGTS_LS |
2102			AMD_CG_SUPPORT_GFX_CP_LS |
2103			AMD_CG_SUPPORT_SDMA_MGCG |
2104			AMD_CG_SUPPORT_SDMA_LS |
2105			AMD_CG_SUPPORT_BIF_LS |
2106			AMD_CG_SUPPORT_VCE_MGCG |
2107			AMD_CG_SUPPORT_UVD_MGCG |
2108			AMD_CG_SUPPORT_HDP_LS |
2109			AMD_CG_SUPPORT_HDP_MGCG;
2110		adev->pg_flags =
2111			/*AMD_PG_SUPPORT_GFX_PG |
2112			  AMD_PG_SUPPORT_GFX_SMG | */
2113			AMD_PG_SUPPORT_UVD |
2114			/*AMD_PG_SUPPORT_VCE |
2115			  AMD_PG_SUPPORT_CP |
2116			  AMD_PG_SUPPORT_GDS |
2117			  AMD_PG_SUPPORT_RLC_SMU_HS |
2118			  AMD_PG_SUPPORT_SAMU |*/
2119			0;
2120		if (adev->asic_type == CHIP_KABINI) {
2121			if (adev->rev_id == 0)
2122				adev->external_rev_id = 0x81;
2123			else if (adev->rev_id == 1)
2124				adev->external_rev_id = 0x82;
2125			else if (adev->rev_id == 2)
2126				adev->external_rev_id = 0x85;
2127		} else
2128			adev->external_rev_id = adev->rev_id + 0xa1;
2129		break;
2130	default:
2131		/* FIXME: not supported yet */
2132		return -EINVAL;
2133	}
2134
 
 
2135	return 0;
2136}
2137
2138static int cik_common_sw_init(void *handle)
2139{
2140	return 0;
2141}
2142
2143static int cik_common_sw_fini(void *handle)
2144{
2145	return 0;
2146}
2147
2148static int cik_common_hw_init(void *handle)
2149{
2150	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2151
2152	/* move the golden regs per IP block */
2153	cik_init_golden_registers(adev);
2154	/* enable pcie gen2/3 link */
2155	cik_pcie_gen3_enable(adev);
2156	/* enable aspm */
2157	cik_program_aspm(adev);
2158
2159	return 0;
2160}
2161
2162static int cik_common_hw_fini(void *handle)
2163{
2164	return 0;
2165}
2166
2167static int cik_common_suspend(void *handle)
2168{
2169	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2170
 
 
2171	return cik_common_hw_fini(adev);
2172}
2173
2174static int cik_common_resume(void *handle)
2175{
 
2176	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2177
2178	return cik_common_hw_init(adev);
 
 
 
 
2179}
2180
2181static bool cik_common_is_idle(void *handle)
2182{
2183	return true;
2184}
2185
2186static int cik_common_wait_for_idle(void *handle)
2187{
2188	return 0;
2189}
2190
2191static int cik_common_soft_reset(void *handle)
2192{
2193	/* XXX hard reset?? */
2194	return 0;
2195}
2196
2197static int cik_common_set_clockgating_state(void *handle,
2198					    enum amd_clockgating_state state)
2199{
2200	return 0;
2201}
2202
2203static int cik_common_set_powergating_state(void *handle,
2204					    enum amd_powergating_state state)
2205{
2206	return 0;
2207}
2208
2209static const struct amd_ip_funcs cik_common_ip_funcs = {
2210	.name = "cik_common",
2211	.early_init = cik_common_early_init,
2212	.late_init = NULL,
2213	.sw_init = cik_common_sw_init,
2214	.sw_fini = cik_common_sw_fini,
2215	.hw_init = cik_common_hw_init,
2216	.hw_fini = cik_common_hw_fini,
2217	.suspend = cik_common_suspend,
2218	.resume = cik_common_resume,
2219	.is_idle = cik_common_is_idle,
2220	.wait_for_idle = cik_common_wait_for_idle,
2221	.soft_reset = cik_common_soft_reset,
2222	.set_clockgating_state = cik_common_set_clockgating_state,
2223	.set_powergating_state = cik_common_set_powergating_state,
2224};
2225
2226static const struct amdgpu_ip_block_version cik_common_ip_block =
2227{
2228	.type = AMD_IP_BLOCK_TYPE_COMMON,
2229	.major = 1,
2230	.minor = 0,
2231	.rev = 0,
2232	.funcs = &cik_common_ip_funcs,
2233};
2234
2235int cik_set_ip_blocks(struct amdgpu_device *adev)
2236{
2237	switch (adev->asic_type) {
2238	case CHIP_BONAIRE:
2239		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2240		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2241		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2242		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2243		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2244		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2245		if (adev->enable_virtual_display)
2246			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2247#if defined(CONFIG_DRM_AMD_DC)
2248		else if (amdgpu_device_has_dc_support(adev))
2249			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2250#endif
2251		else
2252			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2253		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2254		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2255		break;
2256	case CHIP_HAWAII:
2257		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2258		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2259		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2260		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2261		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2262		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2263		if (adev->enable_virtual_display)
2264			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2265#if defined(CONFIG_DRM_AMD_DC)
2266		else if (amdgpu_device_has_dc_support(adev))
2267			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2268#endif
2269		else
2270			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2271		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2272		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2273		break;
2274	case CHIP_KAVERI:
2275		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2276		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2277		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2278		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2279		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2280		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2281		if (adev->enable_virtual_display)
2282			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2283#if defined(CONFIG_DRM_AMD_DC)
2284		else if (amdgpu_device_has_dc_support(adev))
2285			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2286#endif
2287		else
2288			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2289
2290		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2291		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
2292		break;
2293	case CHIP_KABINI:
2294	case CHIP_MULLINS:
2295		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2296		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2297		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2298		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2299		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2300		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2301		if (adev->enable_virtual_display)
2302			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2303#if defined(CONFIG_DRM_AMD_DC)
2304		else if (amdgpu_device_has_dc_support(adev))
2305			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2306#endif
2307		else
2308			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2309		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2310		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 
 
2311		break;
2312	default:
2313		/* FIXME: not supported yet */
2314		return -EINVAL;
2315	}
2316	return 0;
2317}