1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/pci.h>
  25#include <linux/slab.h>
  26
  27#include <drm/amdgpu_drm.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_atombios.h"
  31#include "amdgpu_ih.h"
  32#include "amdgpu_uvd.h"
  33#include "amdgpu_vce.h"
  34#include "amdgpu_ucode.h"
  35#include "atom.h"
  36#include "amd_pcie.h"
  37
  38#include "gmc/gmc_8_1_d.h"
  39#include "gmc/gmc_8_1_sh_mask.h"
  40
  41#include "oss/oss_3_0_d.h"
  42#include "oss/oss_3_0_sh_mask.h"
  43
  44#include "bif/bif_5_0_d.h"
  45#include "bif/bif_5_0_sh_mask.h"
  46
  47#include "gca/gfx_8_0_d.h"
  48#include "gca/gfx_8_0_sh_mask.h"
  49
  50#include "smu/smu_7_1_1_d.h"
  51#include "smu/smu_7_1_1_sh_mask.h"
  52
  53#include "uvd/uvd_5_0_d.h"
  54#include "uvd/uvd_5_0_sh_mask.h"
  55
  56#include "vce/vce_3_0_d.h"
  57#include "vce/vce_3_0_sh_mask.h"
  58
  59#include "dce/dce_10_0_d.h"
  60#include "dce/dce_10_0_sh_mask.h"
  61
  62#include "vid.h"
  63#include "vi.h"
  64#include "gmc_v8_0.h"
  65#include "gmc_v7_0.h"
  66#include "gfx_v8_0.h"
  67#include "sdma_v2_4.h"
  68#include "sdma_v3_0.h"
  69#include "dce_v10_0.h"
  70#include "dce_v11_0.h"
  71#include "iceland_ih.h"
  72#include "tonga_ih.h"
  73#include "cz_ih.h"
  74#include "uvd_v5_0.h"
  75#include "uvd_v6_0.h"
  76#include "vce_v3_0.h"
  77#if defined(CONFIG_DRM_AMD_ACP)
  78#include "amdgpu_acp.h"
  79#endif
  80#include "amdgpu_vkms.h"
  81#include "mxgpu_vi.h"
  82#include "amdgpu_dm.h"
  83
  84#define ixPCIE_LC_L1_PM_SUBSTATE	0x100100C6
  85#define PCIE_LC_L1_PM_SUBSTATE__LC_L1_SUBSTATES_OVERRIDE_EN_MASK	0x00000001L
  86#define PCIE_LC_L1_PM_SUBSTATE__LC_PCI_PM_L1_2_OVERRIDE_MASK	0x00000002L
  87#define PCIE_LC_L1_PM_SUBSTATE__LC_PCI_PM_L1_1_OVERRIDE_MASK	0x00000004L
  88#define PCIE_LC_L1_PM_SUBSTATE__LC_ASPM_L1_2_OVERRIDE_MASK		0x00000008L
  89#define PCIE_LC_L1_PM_SUBSTATE__LC_ASPM_L1_1_OVERRIDE_MASK		0x00000010L
  90#define ixPCIE_L1_PM_SUB_CNTL	0x378
  91#define PCIE_L1_PM_SUB_CNTL__ASPM_L1_2_EN_MASK	0x00000004L
  92#define PCIE_L1_PM_SUB_CNTL__ASPM_L1_1_EN_MASK	0x00000008L
  93#define PCIE_L1_PM_SUB_CNTL__PCI_PM_L1_2_EN_MASK	0x00000001L
  94#define PCIE_L1_PM_SUB_CNTL__PCI_PM_L1_1_EN_MASK	0x00000002L
  95#define PCIE_LC_CNTL6__LC_L1_POWERDOWN_MASK		0x00200000L
  96#define LINK_CAP	0x64
  97#define PCIE_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK	0x00040000L
  98#define ixCPM_CONTROL	0x1400118
  99#define ixPCIE_LC_CNTL7	0x100100BC
 100#define PCIE_LC_CNTL7__LC_L1_SIDEBAND_CLKREQ_PDWN_EN_MASK	0x00000400L
 101#define PCIE_LC_CNTL__LC_L0S_INACTIVITY_DEFAULT	0x00000007
 102#define PCIE_LC_CNTL__LC_L1_INACTIVITY_DEFAULT	0x00000009
 103#define CPM_CONTROL__CLKREQb_UNGATE_TXCLK_ENABLE_MASK	0x01000000L
 104#define PCIE_L1_PM_SUB_CNTL	0x378
 105#define ASIC_IS_P22(asic_type, rid)	((asic_type >= CHIP_POLARIS10) && \
 106									(asic_type <= CHIP_POLARIS12) && \
 107									(rid >= 0x6E))
 108/* Topaz */
 109static const struct amdgpu_video_codecs topaz_video_codecs_encode =
 110{
 111	.codec_count = 0,
 112	.codec_array = NULL,
 113};
 114
 115/* Tonga, CZ, ST, Fiji */
 116static const struct amdgpu_video_codec_info tonga_video_codecs_encode_array[] =
 117{
 118	{
 119		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 120		.max_width = 4096,
 121		.max_height = 2304,
 122		.max_pixels_per_frame = 4096 * 2304,
 123		.max_level = 0,
 124	},
 125};
 126
 127static const struct amdgpu_video_codecs tonga_video_codecs_encode =
 128{
 129	.codec_count = ARRAY_SIZE(tonga_video_codecs_encode_array),
 130	.codec_array = tonga_video_codecs_encode_array,
 131};
 132
 133/* Polaris */
 134static const struct amdgpu_video_codec_info polaris_video_codecs_encode_array[] =
 135{
 136	{
 137		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 138		.max_width = 4096,
 139		.max_height = 2304,
 140		.max_pixels_per_frame = 4096 * 2304,
 141		.max_level = 0,
 142	},
 143	{
 144		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC,
 145		.max_width = 4096,
 146		.max_height = 2304,
 147		.max_pixels_per_frame = 4096 * 2304,
 148		.max_level = 0,
 149	},
 150};
 151
 152static const struct amdgpu_video_codecs polaris_video_codecs_encode =
 153{
 154	.codec_count = ARRAY_SIZE(polaris_video_codecs_encode_array),
 155	.codec_array = polaris_video_codecs_encode_array,
 156};
 157
 158/* Topaz */
 159static const struct amdgpu_video_codecs topaz_video_codecs_decode =
 160{
 161	.codec_count = 0,
 162	.codec_array = NULL,
 163};
 164
 165/* Tonga */
 166static const struct amdgpu_video_codec_info tonga_video_codecs_decode_array[] =
 167{
 168	{
 169		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
 170		.max_width = 4096,
 171		.max_height = 4096,
 172		.max_pixels_per_frame = 4096 * 4096,
 173		.max_level = 3,
 174	},
 175	{
 176		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 177		.max_width = 4096,
 178		.max_height = 4096,
 179		.max_pixels_per_frame = 4096 * 4096,
 180		.max_level = 5,
 181	},
 182	{
 183		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 184		.max_width = 4096,
 185		.max_height = 4096,
 186		.max_pixels_per_frame = 4096 * 4096,
 187		.max_level = 52,
 188	},
 189	{
 190		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 191		.max_width = 4096,
 192		.max_height = 4096,
 193		.max_pixels_per_frame = 4096 * 4096,
 194		.max_level = 4,
 195	},
 196};
 197
 198static const struct amdgpu_video_codecs tonga_video_codecs_decode =
 199{
 200	.codec_count = ARRAY_SIZE(tonga_video_codecs_decode_array),
 201	.codec_array = tonga_video_codecs_decode_array,
 202};
 203
 204/* CZ, ST, Fiji, Polaris */
 205static const struct amdgpu_video_codec_info cz_video_codecs_decode_array[] =
 206{
 207	{
 208		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
 209		.max_width = 4096,
 210		.max_height = 4096,
 211		.max_pixels_per_frame = 4096 * 4096,
 212		.max_level = 3,
 213	},
 214	{
 215		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 216		.max_width = 4096,
 217		.max_height = 4096,
 218		.max_pixels_per_frame = 4096 * 4096,
 219		.max_level = 5,
 220	},
 221	{
 222		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 223		.max_width = 4096,
 224		.max_height = 4096,
 225		.max_pixels_per_frame = 4096 * 4096,
 226		.max_level = 52,
 227	},
 228	{
 229		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 230		.max_width = 4096,
 231		.max_height = 4096,
 232		.max_pixels_per_frame = 4096 * 4096,
 233		.max_level = 4,
 234	},
 235	{
 236		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC,
 237		.max_width = 4096,
 238		.max_height = 4096,
 239		.max_pixels_per_frame = 4096 * 4096,
 240		.max_level = 186,
 241	},
 242	{
 243		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG,
 244		.max_width = 4096,
 245		.max_height = 4096,
 246		.max_pixels_per_frame = 4096 * 4096,
 247		.max_level = 0,
 248	},
 249};
 250
 251static const struct amdgpu_video_codecs cz_video_codecs_decode =
 252{
 253	.codec_count = ARRAY_SIZE(cz_video_codecs_decode_array),
 254	.codec_array = cz_video_codecs_decode_array,
 255};
 256
 257static int vi_query_video_codecs(struct amdgpu_device *adev, bool encode,
 258				 const struct amdgpu_video_codecs **codecs)
 259{
 260	switch (adev->asic_type) {
 261	case CHIP_TOPAZ:
 262		if (encode)
 263			*codecs = &topaz_video_codecs_encode;
 264		else
 265			*codecs = &topaz_video_codecs_decode;
 266		return 0;
 267	case CHIP_TONGA:
 268		if (encode)
 269			*codecs = &tonga_video_codecs_encode;
 270		else
 271			*codecs = &tonga_video_codecs_decode;
 272		return 0;
 273	case CHIP_POLARIS10:
 274	case CHIP_POLARIS11:
 275	case CHIP_POLARIS12:
 276	case CHIP_VEGAM:
 277		if (encode)
 278			*codecs = &polaris_video_codecs_encode;
 279		else
 280			*codecs = &cz_video_codecs_decode;
 281		return 0;
 282	case CHIP_FIJI:
 283	case CHIP_CARRIZO:
 284	case CHIP_STONEY:
 285		if (encode)
 286			*codecs = &tonga_video_codecs_encode;
 287		else
 288			*codecs = &cz_video_codecs_decode;
 289		return 0;
 290	default:
 291		return -EINVAL;
 292	}
 293}
 294
 295/*
 296 * Indirect registers accessor
 297 */
 298static u32 vi_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 299{
 300	unsigned long flags;
 301	u32 r;
 302
 303	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 304	WREG32_NO_KIQ(mmPCIE_INDEX, reg);
 305	(void)RREG32_NO_KIQ(mmPCIE_INDEX);
 306	r = RREG32_NO_KIQ(mmPCIE_DATA);
 307	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 308	return r;
 309}
 310
 311static void vi_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 312{
 313	unsigned long flags;
 314
 315	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 316	WREG32_NO_KIQ(mmPCIE_INDEX, reg);
 317	(void)RREG32_NO_KIQ(mmPCIE_INDEX);
 318	WREG32_NO_KIQ(mmPCIE_DATA, v);
 319	(void)RREG32_NO_KIQ(mmPCIE_DATA);
 320	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 321}
 322
 323static u32 vi_smc_rreg(struct amdgpu_device *adev, u32 reg)
 324{
 325	unsigned long flags;
 326	u32 r;
 327
 328	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 329	WREG32_NO_KIQ(mmSMC_IND_INDEX_11, (reg));
 330	r = RREG32_NO_KIQ(mmSMC_IND_DATA_11);
 331	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 332	return r;
 333}
 334
 335static void vi_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 336{
 337	unsigned long flags;
 338
 339	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 340	WREG32_NO_KIQ(mmSMC_IND_INDEX_11, (reg));
 341	WREG32_NO_KIQ(mmSMC_IND_DATA_11, (v));
 342	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 343}
 344
 345/* smu_8_0_d.h */
 346#define mmMP0PUB_IND_INDEX                                                      0x180
 347#define mmMP0PUB_IND_DATA                                                       0x181
 348
 349static u32 cz_smc_rreg(struct amdgpu_device *adev, u32 reg)
 350{
 351	unsigned long flags;
 352	u32 r;
 353
 354	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 355	WREG32(mmMP0PUB_IND_INDEX, (reg));
 356	r = RREG32(mmMP0PUB_IND_DATA);
 357	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 358	return r;
 359}
 360
 361static void cz_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 362{
 363	unsigned long flags;
 364
 365	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 366	WREG32(mmMP0PUB_IND_INDEX, (reg));
 367	WREG32(mmMP0PUB_IND_DATA, (v));
 368	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 369}
 370
 371static u32 vi_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 372{
 373	unsigned long flags;
 374	u32 r;
 375
 376	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 377	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 378	r = RREG32(mmUVD_CTX_DATA);
 379	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 380	return r;
 381}
 382
 383static void vi_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 384{
 385	unsigned long flags;
 386
 387	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 388	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 389	WREG32(mmUVD_CTX_DATA, (v));
 390	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 391}
 392
 393static u32 vi_didt_rreg(struct amdgpu_device *adev, u32 reg)
 394{
 395	unsigned long flags;
 396	u32 r;
 397
 398	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 399	WREG32(mmDIDT_IND_INDEX, (reg));
 400	r = RREG32(mmDIDT_IND_DATA);
 401	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 402	return r;
 403}
 404
 405static void vi_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 406{
 407	unsigned long flags;
 408
 409	spin_lock_irqsave(&adev->didt_idx_lock, flags);
 410	WREG32(mmDIDT_IND_INDEX, (reg));
 411	WREG32(mmDIDT_IND_DATA, (v));
 412	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 413}
 414
 415static u32 vi_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
 416{
 417	unsigned long flags;
 418	u32 r;
 419
 420	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
 421	WREG32(mmGC_CAC_IND_INDEX, (reg));
 422	r = RREG32(mmGC_CAC_IND_DATA);
 423	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
 424	return r;
 425}
 426
 427static void vi_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 428{
 429	unsigned long flags;
 430
 431	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
 432	WREG32(mmGC_CAC_IND_INDEX, (reg));
 433	WREG32(mmGC_CAC_IND_DATA, (v));
 434	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
 435}
 436
 437
 438static const u32 tonga_mgcg_cgcg_init[] =
 439{
 440	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
 441	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
 442	mmPCIE_DATA, 0x000f0000, 0x00000000,
 443	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
 444	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
 445	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 446	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 447};
 448
 449static const u32 fiji_mgcg_cgcg_init[] =
 450{
 451	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
 452	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
 453	mmPCIE_DATA, 0x000f0000, 0x00000000,
 454	mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
 455	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
 456	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 457	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 458};
 459
 460static const u32 iceland_mgcg_cgcg_init[] =
 461{
 462	mmPCIE_INDEX, 0xffffffff, ixPCIE_CNTL2,
 463	mmPCIE_DATA, 0x000f0000, 0x00000000,
 464	mmSMC_IND_INDEX_4, 0xffffffff, ixCGTT_ROM_CLK_CTRL0,
 465	mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
 466	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 467};
 468
 469static const u32 cz_mgcg_cgcg_init[] =
 470{
 471	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
 472	mmPCIE_INDEX, 0xffffffff, 0x0140001c,
 473	mmPCIE_DATA, 0x000f0000, 0x00000000,
 474	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 475	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 476};
 477
 478static const u32 stoney_mgcg_cgcg_init[] =
 479{
 480	mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00000100,
 481	mmHDP_XDP_CGTT_BLK_CTRL, 0xffffffff, 0x00000104,
 482	mmHDP_HOST_PATH_CNTL, 0xffffffff, 0x0f000027,
 483};
 484
 485static void vi_init_golden_registers(struct amdgpu_device *adev)
 486{
 487	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
 488	mutex_lock(&adev->grbm_idx_mutex);
 489
 490	if (amdgpu_sriov_vf(adev)) {
 491		xgpu_vi_init_golden_registers(adev);
 492		mutex_unlock(&adev->grbm_idx_mutex);
 493		return;
 494	}
 495
 496	switch (adev->asic_type) {
 497	case CHIP_TOPAZ:
 498		amdgpu_device_program_register_sequence(adev,
 499							iceland_mgcg_cgcg_init,
 500							ARRAY_SIZE(iceland_mgcg_cgcg_init));
 501		break;
 502	case CHIP_FIJI:
 503		amdgpu_device_program_register_sequence(adev,
 504							fiji_mgcg_cgcg_init,
 505							ARRAY_SIZE(fiji_mgcg_cgcg_init));
 506		break;
 507	case CHIP_TONGA:
 508		amdgpu_device_program_register_sequence(adev,
 509							tonga_mgcg_cgcg_init,
 510							ARRAY_SIZE(tonga_mgcg_cgcg_init));
 511		break;
 512	case CHIP_CARRIZO:
 513		amdgpu_device_program_register_sequence(adev,
 514							cz_mgcg_cgcg_init,
 515							ARRAY_SIZE(cz_mgcg_cgcg_init));
 516		break;
 517	case CHIP_STONEY:
 518		amdgpu_device_program_register_sequence(adev,
 519							stoney_mgcg_cgcg_init,
 520							ARRAY_SIZE(stoney_mgcg_cgcg_init));
 521		break;
 522	case CHIP_POLARIS10:
 523	case CHIP_POLARIS11:
 524	case CHIP_POLARIS12:
 525	case CHIP_VEGAM:
 526	default:
 527		break;
 528	}
 529	mutex_unlock(&adev->grbm_idx_mutex);
 530}
 531
 532/**
 533 * vi_get_xclk - get the xclk
 534 *
 535 * @adev: amdgpu_device pointer
 536 *
 537 * Returns the reference clock used by the gfx engine
 538 * (VI).
 539 */
 540static u32 vi_get_xclk(struct amdgpu_device *adev)
 541{
 542	u32 reference_clock = adev->clock.spll.reference_freq;
 543	u32 tmp;
 544
 545	if (adev->flags & AMD_IS_APU) {
 546		switch (adev->asic_type) {
 547		case CHIP_STONEY:
 548			/* vbios says 48Mhz, but the actual freq is 100Mhz */
 549			return 10000;
 550		default:
 551			return reference_clock;
 552		}
 553	}
 554
 555	tmp = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
 556	if (REG_GET_FIELD(tmp, CG_CLKPIN_CNTL_2, MUX_TCLK_TO_XCLK))
 557		return 1000;
 558
 559	tmp = RREG32_SMC(ixCG_CLKPIN_CNTL);
 560	if (REG_GET_FIELD(tmp, CG_CLKPIN_CNTL, XTALIN_DIVIDE))
 561		return reference_clock / 4;
 562
 563	return reference_clock;
 564}
 565
 566/**
 567 * vi_srbm_select - select specific register instances
 568 *
 569 * @adev: amdgpu_device pointer
 570 * @me: selected ME (micro engine)
 571 * @pipe: pipe
 572 * @queue: queue
 573 * @vmid: VMID
 574 *
 575 * Switches the currently active registers instances.  Some
 576 * registers are instanced per VMID, others are instanced per
 577 * me/pipe/queue combination.
 578 */
 579void vi_srbm_select(struct amdgpu_device *adev,
 580		     u32 me, u32 pipe, u32 queue, u32 vmid)
 581{
 582	u32 srbm_gfx_cntl = 0;
 583	srbm_gfx_cntl = REG_SET_FIELD(srbm_gfx_cntl, SRBM_GFX_CNTL, PIPEID, pipe);
 584	srbm_gfx_cntl = REG_SET_FIELD(srbm_gfx_cntl, SRBM_GFX_CNTL, MEID, me);
 585	srbm_gfx_cntl = REG_SET_FIELD(srbm_gfx_cntl, SRBM_GFX_CNTL, VMID, vmid);
 586	srbm_gfx_cntl = REG_SET_FIELD(srbm_gfx_cntl, SRBM_GFX_CNTL, QUEUEID, queue);
 587	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 588}
 589
 
 
 
 
 
 590static bool vi_read_disabled_bios(struct amdgpu_device *adev)
 591{
 592	u32 bus_cntl;
 593	u32 d1vga_control = 0;
 594	u32 d2vga_control = 0;
 595	u32 vga_render_control = 0;
 596	u32 rom_cntl;
 597	bool r;
 598
 599	bus_cntl = RREG32(mmBUS_CNTL);
 600	if (adev->mode_info.num_crtc) {
 601		d1vga_control = RREG32(mmD1VGA_CONTROL);
 602		d2vga_control = RREG32(mmD2VGA_CONTROL);
 603		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 604	}
 605	rom_cntl = RREG32_SMC(ixROM_CNTL);
 606
 607	/* enable the rom */
 608	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 609	if (adev->mode_info.num_crtc) {
 610		/* Disable VGA mode */
 611		WREG32(mmD1VGA_CONTROL,
 612		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 613					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 614		WREG32(mmD2VGA_CONTROL,
 615		       (d2vga_control & ~(D2VGA_CONTROL__D2VGA_MODE_ENABLE_MASK |
 616					  D2VGA_CONTROL__D2VGA_TIMING_SELECT_MASK)));
 617		WREG32(mmVGA_RENDER_CONTROL,
 618		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 619	}
 620	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 621
 622	r = amdgpu_read_bios(adev);
 623
 624	/* restore regs */
 625	WREG32(mmBUS_CNTL, bus_cntl);
 626	if (adev->mode_info.num_crtc) {
 627		WREG32(mmD1VGA_CONTROL, d1vga_control);
 628		WREG32(mmD2VGA_CONTROL, d2vga_control);
 629		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 630	}
 631	WREG32_SMC(ixROM_CNTL, rom_cntl);
 632	return r;
 633}
 634
 635static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 636				  u8 *bios, u32 length_bytes)
 637{
 638	u32 *dw_ptr;
 639	unsigned long flags;
 640	u32 i, length_dw;
 641
 642	if (bios == NULL)
 643		return false;
 644	if (length_bytes == 0)
 645		return false;
 646	/* APU vbios image is part of sbios image */
 647	if (adev->flags & AMD_IS_APU)
 648		return false;
 649
 650	dw_ptr = (u32 *)bios;
 651	length_dw = ALIGN(length_bytes, 4) / 4;
 652	/* take the smc lock since we are using the smc index */
 653	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 654	/* set rom index to 0 */
 655	WREG32(mmSMC_IND_INDEX_11, ixROM_INDEX);
 656	WREG32(mmSMC_IND_DATA_11, 0);
 657	/* set index to data for continous read */
 658	WREG32(mmSMC_IND_INDEX_11, ixROM_DATA);
 659	for (i = 0; i < length_dw; i++)
 660		dw_ptr[i] = RREG32(mmSMC_IND_DATA_11);
 661	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 662
 663	return true;
 664}
 665
 666static const struct amdgpu_allowed_register_entry vi_allowed_read_registers[] = {
 667	{mmGRBM_STATUS},
 668	{mmGRBM_STATUS2},
 669	{mmGRBM_STATUS_SE0},
 670	{mmGRBM_STATUS_SE1},
 671	{mmGRBM_STATUS_SE2},
 672	{mmGRBM_STATUS_SE3},
 673	{mmSRBM_STATUS},
 674	{mmSRBM_STATUS2},
 675	{mmSRBM_STATUS3},
 676	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
 677	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
 678	{mmCP_STAT},
 679	{mmCP_STALLED_STAT1},
 680	{mmCP_STALLED_STAT2},
 681	{mmCP_STALLED_STAT3},
 682	{mmCP_CPF_BUSY_STAT},
 683	{mmCP_CPF_STALLED_STAT1},
 684	{mmCP_CPF_STATUS},
 685	{mmCP_CPC_BUSY_STAT},
 686	{mmCP_CPC_STALLED_STAT1},
 687	{mmCP_CPC_STATUS},
 688	{mmGB_ADDR_CONFIG},
 689	{mmMC_ARB_RAMCFG},
 690	{mmGB_TILE_MODE0},
 691	{mmGB_TILE_MODE1},
 692	{mmGB_TILE_MODE2},
 693	{mmGB_TILE_MODE3},
 694	{mmGB_TILE_MODE4},
 695	{mmGB_TILE_MODE5},
 696	{mmGB_TILE_MODE6},
 697	{mmGB_TILE_MODE7},
 698	{mmGB_TILE_MODE8},
 699	{mmGB_TILE_MODE9},
 700	{mmGB_TILE_MODE10},
 701	{mmGB_TILE_MODE11},
 702	{mmGB_TILE_MODE12},
 703	{mmGB_TILE_MODE13},
 704	{mmGB_TILE_MODE14},
 705	{mmGB_TILE_MODE15},
 706	{mmGB_TILE_MODE16},
 707	{mmGB_TILE_MODE17},
 708	{mmGB_TILE_MODE18},
 709	{mmGB_TILE_MODE19},
 710	{mmGB_TILE_MODE20},
 711	{mmGB_TILE_MODE21},
 712	{mmGB_TILE_MODE22},
 713	{mmGB_TILE_MODE23},
 714	{mmGB_TILE_MODE24},
 715	{mmGB_TILE_MODE25},
 716	{mmGB_TILE_MODE26},
 717	{mmGB_TILE_MODE27},
 718	{mmGB_TILE_MODE28},
 719	{mmGB_TILE_MODE29},
 720	{mmGB_TILE_MODE30},
 721	{mmGB_TILE_MODE31},
 722	{mmGB_MACROTILE_MODE0},
 723	{mmGB_MACROTILE_MODE1},
 724	{mmGB_MACROTILE_MODE2},
 725	{mmGB_MACROTILE_MODE3},
 726	{mmGB_MACROTILE_MODE4},
 727	{mmGB_MACROTILE_MODE5},
 728	{mmGB_MACROTILE_MODE6},
 729	{mmGB_MACROTILE_MODE7},
 730	{mmGB_MACROTILE_MODE8},
 731	{mmGB_MACROTILE_MODE9},
 732	{mmGB_MACROTILE_MODE10},
 733	{mmGB_MACROTILE_MODE11},
 734	{mmGB_MACROTILE_MODE12},
 735	{mmGB_MACROTILE_MODE13},
 736	{mmGB_MACROTILE_MODE14},
 737	{mmGB_MACROTILE_MODE15},
 738	{mmCC_RB_BACKEND_DISABLE, true},
 739	{mmGC_USER_RB_BACKEND_DISABLE, true},
 740	{mmGB_BACKEND_MAP, false},
 741	{mmPA_SC_RASTER_CONFIG, true},
 742	{mmPA_SC_RASTER_CONFIG_1, true},
 743};
 744
 745static uint32_t vi_get_register_value(struct amdgpu_device *adev,
 746				      bool indexed, u32 se_num,
 747				      u32 sh_num, u32 reg_offset)
 748{
 749	if (indexed) {
 750		uint32_t val;
 751		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
 752		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
 753
 754		switch (reg_offset) {
 755		case mmCC_RB_BACKEND_DISABLE:
 756			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
 757		case mmGC_USER_RB_BACKEND_DISABLE:
 758			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
 759		case mmPA_SC_RASTER_CONFIG:
 760			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
 761		case mmPA_SC_RASTER_CONFIG_1:
 762			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
 763		}
 764
 765		mutex_lock(&adev->grbm_idx_mutex);
 766		if (se_num != 0xffffffff || sh_num != 0xffffffff)
 767			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
 768
 769		val = RREG32(reg_offset);
 770
 771		if (se_num != 0xffffffff || sh_num != 0xffffffff)
 772			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
 773		mutex_unlock(&adev->grbm_idx_mutex);
 774		return val;
 775	} else {
 776		unsigned idx;
 777
 778		switch (reg_offset) {
 779		case mmGB_ADDR_CONFIG:
 780			return adev->gfx.config.gb_addr_config;
 781		case mmMC_ARB_RAMCFG:
 782			return adev->gfx.config.mc_arb_ramcfg;
 783		case mmGB_TILE_MODE0:
 784		case mmGB_TILE_MODE1:
 785		case mmGB_TILE_MODE2:
 786		case mmGB_TILE_MODE3:
 787		case mmGB_TILE_MODE4:
 788		case mmGB_TILE_MODE5:
 789		case mmGB_TILE_MODE6:
 790		case mmGB_TILE_MODE7:
 791		case mmGB_TILE_MODE8:
 792		case mmGB_TILE_MODE9:
 793		case mmGB_TILE_MODE10:
 794		case mmGB_TILE_MODE11:
 795		case mmGB_TILE_MODE12:
 796		case mmGB_TILE_MODE13:
 797		case mmGB_TILE_MODE14:
 798		case mmGB_TILE_MODE15:
 799		case mmGB_TILE_MODE16:
 800		case mmGB_TILE_MODE17:
 801		case mmGB_TILE_MODE18:
 802		case mmGB_TILE_MODE19:
 803		case mmGB_TILE_MODE20:
 804		case mmGB_TILE_MODE21:
 805		case mmGB_TILE_MODE22:
 806		case mmGB_TILE_MODE23:
 807		case mmGB_TILE_MODE24:
 808		case mmGB_TILE_MODE25:
 809		case mmGB_TILE_MODE26:
 810		case mmGB_TILE_MODE27:
 811		case mmGB_TILE_MODE28:
 812		case mmGB_TILE_MODE29:
 813		case mmGB_TILE_MODE30:
 814		case mmGB_TILE_MODE31:
 815			idx = (reg_offset - mmGB_TILE_MODE0);
 816			return adev->gfx.config.tile_mode_array[idx];
 817		case mmGB_MACROTILE_MODE0:
 818		case mmGB_MACROTILE_MODE1:
 819		case mmGB_MACROTILE_MODE2:
 820		case mmGB_MACROTILE_MODE3:
 821		case mmGB_MACROTILE_MODE4:
 822		case mmGB_MACROTILE_MODE5:
 823		case mmGB_MACROTILE_MODE6:
 824		case mmGB_MACROTILE_MODE7:
 825		case mmGB_MACROTILE_MODE8:
 826		case mmGB_MACROTILE_MODE9:
 827		case mmGB_MACROTILE_MODE10:
 828		case mmGB_MACROTILE_MODE11:
 829		case mmGB_MACROTILE_MODE12:
 830		case mmGB_MACROTILE_MODE13:
 831		case mmGB_MACROTILE_MODE14:
 832		case mmGB_MACROTILE_MODE15:
 833			idx = (reg_offset - mmGB_MACROTILE_MODE0);
 834			return adev->gfx.config.macrotile_mode_array[idx];
 835		default:
 836			return RREG32(reg_offset);
 837		}
 838	}
 839}
 840
 841static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
 842			    u32 sh_num, u32 reg_offset, u32 *value)
 843{
 844	uint32_t i;
 845
 846	*value = 0;
 847	for (i = 0; i < ARRAY_SIZE(vi_allowed_read_registers); i++) {
 848		bool indexed = vi_allowed_read_registers[i].grbm_indexed;
 849
 850		if (reg_offset != vi_allowed_read_registers[i].reg_offset)
 851			continue;
 852
 853		*value = vi_get_register_value(adev, indexed, se_num, sh_num,
 854					       reg_offset);
 855		return 0;
 856	}
 857	return -EINVAL;
 858}
 859
 860/**
 861 * vi_asic_pci_config_reset - soft reset GPU
 862 *
 863 * @adev: amdgpu_device pointer
 864 *
 865 * Use PCI Config method to reset the GPU.
 866 *
 867 * Returns 0 for success.
 868 */
 869static int vi_asic_pci_config_reset(struct amdgpu_device *adev)
 870{
 871	u32 i;
 872	int r = -EINVAL;
 873
 874	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
 875
 876	/* disable BM */
 877	pci_clear_master(adev->pdev);
 878	/* reset */
 879	amdgpu_device_pci_config_reset(adev);
 880
 881	udelay(100);
 882
 883	/* wait for asic to come out of reset */
 884	for (i = 0; i < adev->usec_timeout; i++) {
 885		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
 886			/* enable BM */
 887			pci_set_master(adev->pdev);
 888			adev->has_hw_reset = true;
 889			r = 0;
 890			break;
 891		}
 892		udelay(1);
 893	}
 894
 895	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 896
 897	return r;
 898}
 899
 900static bool vi_asic_supports_baco(struct amdgpu_device *adev)
 901{
 902	switch (adev->asic_type) {
 903	case CHIP_FIJI:
 904	case CHIP_TONGA:
 905	case CHIP_POLARIS10:
 906	case CHIP_POLARIS11:
 907	case CHIP_POLARIS12:
 908	case CHIP_TOPAZ:
 909		return amdgpu_dpm_is_baco_supported(adev);
 910	default:
 911		return false;
 912	}
 913}
 914
 915static enum amd_reset_method
 916vi_asic_reset_method(struct amdgpu_device *adev)
 917{
 918	bool baco_reset;
 919
 920	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
 921	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
 922		return amdgpu_reset_method;
 923
 924	if (amdgpu_reset_method != -1)
 925		dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
 926				  amdgpu_reset_method);
 927
 928	switch (adev->asic_type) {
 929	case CHIP_FIJI:
 930	case CHIP_TONGA:
 931	case CHIP_POLARIS10:
 932	case CHIP_POLARIS11:
 933	case CHIP_POLARIS12:
 934	case CHIP_TOPAZ:
 935		baco_reset = amdgpu_dpm_is_baco_supported(adev);
 936		break;
 937	default:
 938		baco_reset = false;
 939		break;
 940	}
 941
 942	if (baco_reset)
 943		return AMD_RESET_METHOD_BACO;
 944	else
 945		return AMD_RESET_METHOD_LEGACY;
 946}
 947
 948/**
 949 * vi_asic_reset - soft reset GPU
 950 *
 951 * @adev: amdgpu_device pointer
 952 *
 953 * Look up which blocks are hung and attempt
 954 * to reset them.
 955 * Returns 0 for success.
 956 */
 957static int vi_asic_reset(struct amdgpu_device *adev)
 958{
 959	int r;
 960
 961	/* APUs don't have full asic reset */
 962	if (adev->flags & AMD_IS_APU)
 963		return 0;
 964
 965	if (vi_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
 966		dev_info(adev->dev, "BACO reset\n");
 967		r = amdgpu_dpm_baco_reset(adev);
 968	} else {
 969		dev_info(adev->dev, "PCI CONFIG reset\n");
 970		r = vi_asic_pci_config_reset(adev);
 971	}
 972
 973	return r;
 974}
 975
 976static u32 vi_get_config_memsize(struct amdgpu_device *adev)
 977{
 978	return RREG32(mmCONFIG_MEMSIZE);
 979}
 980
 981static int vi_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
 982			u32 cntl_reg, u32 status_reg)
 983{
 984	int r, i;
 985	struct atom_clock_dividers dividers;
 986	uint32_t tmp;
 987
 988	r = amdgpu_atombios_get_clock_dividers(adev,
 989					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
 990					       clock, false, &dividers);
 991	if (r)
 992		return r;
 993
 994	tmp = RREG32_SMC(cntl_reg);
 995
 996	if (adev->flags & AMD_IS_APU)
 997		tmp &= ~CG_DCLK_CNTL__DCLK_DIVIDER_MASK;
 998	else
 999		tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1000				CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1001	tmp |= dividers.post_divider;
1002	WREG32_SMC(cntl_reg, tmp);
1003
1004	for (i = 0; i < 100; i++) {
1005		tmp = RREG32_SMC(status_reg);
1006		if (adev->flags & AMD_IS_APU) {
1007			if (tmp & 0x10000)
1008				break;
1009		} else {
1010			if (tmp & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1011				break;
1012		}
1013		mdelay(10);
1014	}
1015	if (i == 100)
1016		return -ETIMEDOUT;
1017	return 0;
1018}
1019
1020#define ixGNB_CLK1_DFS_CNTL 0xD82200F0
1021#define ixGNB_CLK1_STATUS   0xD822010C
1022#define ixGNB_CLK2_DFS_CNTL 0xD8220110
1023#define ixGNB_CLK2_STATUS   0xD822012C
1024#define ixGNB_CLK3_DFS_CNTL 0xD8220130
1025#define ixGNB_CLK3_STATUS   0xD822014C
1026
1027static int vi_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1028{
1029	int r;
1030
1031	if (adev->flags & AMD_IS_APU) {
1032		r = vi_set_uvd_clock(adev, vclk, ixGNB_CLK2_DFS_CNTL, ixGNB_CLK2_STATUS);
1033		if (r)
1034			return r;
1035
1036		r = vi_set_uvd_clock(adev, dclk, ixGNB_CLK1_DFS_CNTL, ixGNB_CLK1_STATUS);
1037		if (r)
1038			return r;
1039	} else {
1040		r = vi_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1041		if (r)
1042			return r;
1043
1044		r = vi_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1045		if (r)
1046			return r;
1047	}
1048
1049	return 0;
1050}
1051
1052static int vi_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1053{
1054	int r, i;
1055	struct atom_clock_dividers dividers;
1056	u32 tmp;
1057	u32 reg_ctrl;
1058	u32 reg_status;
1059	u32 status_mask;
1060	u32 reg_mask;
1061
1062	if (adev->flags & AMD_IS_APU) {
1063		reg_ctrl = ixGNB_CLK3_DFS_CNTL;
1064		reg_status = ixGNB_CLK3_STATUS;
1065		status_mask = 0x00010000;
1066		reg_mask = CG_ECLK_CNTL__ECLK_DIVIDER_MASK;
1067	} else {
1068		reg_ctrl = ixCG_ECLK_CNTL;
1069		reg_status = ixCG_ECLK_STATUS;
1070		status_mask = CG_ECLK_STATUS__ECLK_STATUS_MASK;
1071		reg_mask = CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK | CG_ECLK_CNTL__ECLK_DIVIDER_MASK;
1072	}
1073
1074	r = amdgpu_atombios_get_clock_dividers(adev,
1075					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1076					       ecclk, false, &dividers);
1077	if (r)
1078		return r;
1079
1080	for (i = 0; i < 100; i++) {
1081		if (RREG32_SMC(reg_status) & status_mask)
1082			break;
1083		mdelay(10);
1084	}
1085
1086	if (i == 100)
1087		return -ETIMEDOUT;
1088
1089	tmp = RREG32_SMC(reg_ctrl);
1090	tmp &= ~reg_mask;
1091	tmp |= dividers.post_divider;
1092	WREG32_SMC(reg_ctrl, tmp);
1093
1094	for (i = 0; i < 100; i++) {
1095		if (RREG32_SMC(reg_status) & status_mask)
1096			break;
1097		mdelay(10);
1098	}
1099
1100	if (i == 100)
1101		return -ETIMEDOUT;
1102
1103	return 0;
1104}
1105
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1106static void vi_enable_aspm(struct amdgpu_device *adev)
1107{
1108	u32 data, orig;
1109
1110	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1111	data |= PCIE_LC_CNTL__LC_L0S_INACTIVITY_DEFAULT <<
1112			PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT;
1113	data |= PCIE_LC_CNTL__LC_L1_INACTIVITY_DEFAULT <<
1114			PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT;
1115	data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1116	data |= PCIE_LC_CNTL__LC_DELAY_L1_EXIT_MASK;
1117	if (orig != data)
1118		WREG32_PCIE(ixPCIE_LC_CNTL, data);
1119}
1120
1121static void vi_program_aspm(struct amdgpu_device *adev)
1122{
1123	u32 data, data1, orig;
1124	bool bL1SS = false;
1125	bool bClkReqSupport = true;
1126
1127	if (!amdgpu_device_should_use_aspm(adev))
1128		return;
1129
1130	if (adev->asic_type < CHIP_POLARIS10)
 
1131		return;
1132
1133	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1134	data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK;
1135	data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1136	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1137	if (orig != data)
1138		WREG32_PCIE(ixPCIE_LC_CNTL, data);
1139
1140	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1141	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1142	data |= 0x0024 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT;
1143	data |= PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1144	if (orig != data)
1145		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1146
1147	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1148	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1149	if (orig != data)
1150		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1151
1152	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1153	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1154	if (orig != data)
1155		WREG32_PCIE(ixPCIE_P_CNTL, data);
1156
1157	data = RREG32_PCIE(ixPCIE_LC_L1_PM_SUBSTATE);
1158	pci_read_config_dword(adev->pdev, PCIE_L1_PM_SUB_CNTL, &data1);
1159	if (data & PCIE_LC_L1_PM_SUBSTATE__LC_L1_SUBSTATES_OVERRIDE_EN_MASK &&
1160	    (data & (PCIE_LC_L1_PM_SUBSTATE__LC_PCI_PM_L1_2_OVERRIDE_MASK |
1161		    PCIE_LC_L1_PM_SUBSTATE__LC_PCI_PM_L1_1_OVERRIDE_MASK |
1162			PCIE_LC_L1_PM_SUBSTATE__LC_ASPM_L1_2_OVERRIDE_MASK |
1163			PCIE_LC_L1_PM_SUBSTATE__LC_ASPM_L1_1_OVERRIDE_MASK))) {
1164		bL1SS = true;
1165	} else if (data1 & (PCIE_L1_PM_SUB_CNTL__ASPM_L1_2_EN_MASK |
1166	    PCIE_L1_PM_SUB_CNTL__ASPM_L1_1_EN_MASK |
1167	    PCIE_L1_PM_SUB_CNTL__PCI_PM_L1_2_EN_MASK |
1168	    PCIE_L1_PM_SUB_CNTL__PCI_PM_L1_1_EN_MASK)) {
1169		bL1SS = true;
1170	}
1171
1172	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL6);
1173	data |= PCIE_LC_CNTL6__LC_L1_POWERDOWN_MASK;
1174	if (orig != data)
1175		WREG32_PCIE(ixPCIE_LC_CNTL6, data);
1176
1177	orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1178	data |= PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1179	if (orig != data)
1180		WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1181
1182	pci_read_config_dword(adev->pdev, LINK_CAP, &data);
1183	if (!(data & PCIE_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK))
1184		bClkReqSupport = false;
1185
1186	if (bClkReqSupport) {
1187		orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1188		data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK | THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1189		data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1190				(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1191		if (orig != data)
1192			WREG32_SMC(ixTHM_CLK_CNTL, data);
1193
1194		orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1195		data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1196			MISC_CLK_CTRL__ZCLK_SEL_MASK | MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL_MASK);
1197		data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1198				(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1199		data |= (0x20 << MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL__SHIFT);
1200		if (orig != data)
1201			WREG32_SMC(ixMISC_CLK_CTRL, data);
1202
1203		orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1204		data |= CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK;
1205		if (orig != data)
1206			WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1207
1208		orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1209		data |= CG_CLKPIN_CNTL_2__ENABLE_XCLK_MASK;
1210		if (orig != data)
1211			WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1212
1213		orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1214		data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1215		data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1216		if (orig != data)
1217			WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1218
1219		orig = data = RREG32_PCIE(ixCPM_CONTROL);
1220		data |= (CPM_CONTROL__REFCLK_XSTCLK_ENABLE_MASK |
1221				CPM_CONTROL__CLKREQb_UNGATE_TXCLK_ENABLE_MASK);
1222		if (orig != data)
1223			WREG32_PCIE(ixCPM_CONTROL, data);
1224
1225		orig = data = RREG32_PCIE(ixPCIE_CONFIG_CNTL);
1226		data &= ~PCIE_CONFIG_CNTL__DYN_CLK_LATENCY_MASK;
1227		data |= (0xE << PCIE_CONFIG_CNTL__DYN_CLK_LATENCY__SHIFT);
1228		if (orig != data)
1229			WREG32_PCIE(ixPCIE_CONFIG_CNTL, data);
1230
1231		orig = data = RREG32(mmBIF_CLK_CTRL);
1232		data |= BIF_CLK_CTRL__BIF_XSTCLK_READY_MASK;
1233		if (orig != data)
1234			WREG32(mmBIF_CLK_CTRL, data);
1235
1236		orig = data = RREG32_PCIE(ixPCIE_LC_CNTL7);
1237		data |= PCIE_LC_CNTL7__LC_L1_SIDEBAND_CLKREQ_PDWN_EN_MASK;
1238		if (orig != data)
1239			WREG32_PCIE(ixPCIE_LC_CNTL7, data);
1240
1241		orig = data = RREG32_PCIE(ixPCIE_HW_DEBUG);
1242		data |= PCIE_HW_DEBUG__HW_01_DEBUG_MASK;
1243		if (orig != data)
1244			WREG32_PCIE(ixPCIE_HW_DEBUG, data);
1245
1246		orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1247		data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1248		data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK;
1249		if (bL1SS)
1250			data &= ~PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK;
1251		if (orig != data)
1252			WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1253
1254	}
1255
1256	vi_enable_aspm(adev);
1257
1258	data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1259	data1 = RREG32_PCIE(ixPCIE_LC_STATUS1);
1260	if (((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) == PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) &&
1261	    data1 & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK &&
1262	    data1 & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK) {
1263		orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1264		data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1265		if (orig != data)
1266			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1267	}
1268
1269	if ((adev->asic_type == CHIP_POLARIS12 &&
1270	    !(ASICID_IS_P23(adev->pdev->device, adev->pdev->revision))) ||
1271	    ASIC_IS_P22(adev->asic_type, adev->external_rev_id)) {
1272		orig = data = RREG32_PCIE(ixPCIE_LC_TRAINING_CNTL);
1273		data &= ~PCIE_LC_TRAINING_CNTL__LC_DISABLE_TRAINING_BIT_ARCH_MASK;
1274		if (orig != data)
1275			WREG32_PCIE(ixPCIE_LC_TRAINING_CNTL, data);
1276	}
1277}
1278
1279static void vi_enable_doorbell_aperture(struct amdgpu_device *adev,
1280					bool enable)
1281{
1282	u32 tmp;
1283
1284	/* not necessary on CZ */
1285	if (adev->flags & AMD_IS_APU)
1286		return;
1287
1288	tmp = RREG32(mmBIF_DOORBELL_APER_EN);
1289	if (enable)
1290		tmp = REG_SET_FIELD(tmp, BIF_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, 1);
1291	else
1292		tmp = REG_SET_FIELD(tmp, BIF_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, 0);
1293
1294	WREG32(mmBIF_DOORBELL_APER_EN, tmp);
1295}
1296
1297#define ATI_REV_ID_FUSE_MACRO__ADDRESS      0xC0014044
1298#define ATI_REV_ID_FUSE_MACRO__SHIFT        9
1299#define ATI_REV_ID_FUSE_MACRO__MASK         0x00001E00
1300
1301static uint32_t vi_get_rev_id(struct amdgpu_device *adev)
1302{
1303	if (adev->flags & AMD_IS_APU)
1304		return (RREG32_SMC(ATI_REV_ID_FUSE_MACRO__ADDRESS) & ATI_REV_ID_FUSE_MACRO__MASK)
1305			>> ATI_REV_ID_FUSE_MACRO__SHIFT;
1306	else
1307		return (RREG32(mmPCIE_EFUSE4) & PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID_MASK)
1308			>> PCIE_EFUSE4__STRAP_BIF_ATI_REV_ID__SHIFT;
1309}
1310
1311static void vi_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1312{
1313	if (!ring || !ring->funcs->emit_wreg) {
1314		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1315		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1316	} else {
1317		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1318	}
1319}
1320
1321static void vi_invalidate_hdp(struct amdgpu_device *adev,
1322			      struct amdgpu_ring *ring)
1323{
1324	if (!ring || !ring->funcs->emit_wreg) {
1325		WREG32(mmHDP_DEBUG0, 1);
1326		RREG32(mmHDP_DEBUG0);
1327	} else {
1328		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1329	}
1330}
1331
1332static bool vi_need_full_reset(struct amdgpu_device *adev)
1333{
1334	switch (adev->asic_type) {
1335	case CHIP_CARRIZO:
1336	case CHIP_STONEY:
1337		/* CZ has hang issues with full reset at the moment */
1338		return false;
1339	case CHIP_FIJI:
1340	case CHIP_TONGA:
1341		/* XXX: soft reset should work on fiji and tonga */
1342		return true;
1343	case CHIP_POLARIS10:
1344	case CHIP_POLARIS11:
1345	case CHIP_POLARIS12:
1346	case CHIP_TOPAZ:
1347	default:
1348		/* change this when we support soft reset */
1349		return true;
1350	}
1351}
1352
1353static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1354			      uint64_t *count1)
1355{
1356	uint32_t perfctr = 0;
1357	uint64_t cnt0_of, cnt1_of;
1358	int tmp;
1359
1360	/* This reports 0 on APUs, so return to avoid writing/reading registers
1361	 * that may or may not be different from their GPU counterparts
1362	 */
1363	if (adev->flags & AMD_IS_APU)
1364		return;
1365
1366	/* Set the 2 events that we wish to watch, defined above */
1367	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1368	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1369	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1370
1371	/* Write to enable desired perf counters */
1372	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1373	/* Zero out and enable the perf counters
1374	 * Write 0x5:
1375	 * Bit 0 = Start all counters(1)
1376	 * Bit 2 = Global counter reset enable(1)
1377	 */
1378	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1379
1380	msleep(1000);
1381
1382	/* Load the shadow and disable the perf counters
1383	 * Write 0x2:
1384	 * Bit 0 = Stop counters(0)
1385	 * Bit 1 = Load the shadow counters(1)
1386	 */
1387	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1388
1389	/* Read register values to get any >32bit overflow */
1390	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1391	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1392	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1393
1394	/* Get the values and add the overflow */
1395	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1396	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1397}
1398
1399static uint64_t vi_get_pcie_replay_count(struct amdgpu_device *adev)
1400{
1401	uint64_t nak_r, nak_g;
1402
1403	/* Get the number of NAKs received and generated */
1404	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1405	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1406
1407	/* Add the total number of NAKs, i.e the number of replays */
1408	return (nak_r + nak_g);
1409}
1410
1411static bool vi_need_reset_on_init(struct amdgpu_device *adev)
1412{
1413	u32 clock_cntl, pc;
1414
1415	if (adev->flags & AMD_IS_APU)
1416		return false;
1417
1418	/* check if the SMC is already running */
1419	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1420	pc = RREG32_SMC(ixSMC_PC_C);
1421	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1422	    (0x20100 <= pc))
1423		return true;
1424
1425	return false;
1426}
1427
1428static void vi_pre_asic_init(struct amdgpu_device *adev)
1429{
1430}
1431
1432static const struct amdgpu_asic_funcs vi_asic_funcs =
1433{
1434	.read_disabled_bios = &vi_read_disabled_bios,
1435	.read_bios_from_rom = &vi_read_bios_from_rom,
1436	.read_register = &vi_read_register,
1437	.reset = &vi_asic_reset,
1438	.reset_method = &vi_asic_reset_method,
 
1439	.get_xclk = &vi_get_xclk,
1440	.set_uvd_clocks = &vi_set_uvd_clocks,
1441	.set_vce_clocks = &vi_set_vce_clocks,
1442	.get_config_memsize = &vi_get_config_memsize,
1443	.flush_hdp = &vi_flush_hdp,
1444	.invalidate_hdp = &vi_invalidate_hdp,
1445	.need_full_reset = &vi_need_full_reset,
1446	.init_doorbell_index = &legacy_doorbell_index_init,
1447	.get_pcie_usage = &vi_get_pcie_usage,
1448	.need_reset_on_init = &vi_need_reset_on_init,
1449	.get_pcie_replay_count = &vi_get_pcie_replay_count,
1450	.supports_baco = &vi_asic_supports_baco,
1451	.pre_asic_init = &vi_pre_asic_init,
1452	.query_video_codecs = &vi_query_video_codecs,
1453};
1454
1455#define CZ_REV_BRISTOL(rev)	 \
1456	((rev >= 0xC8 && rev <= 0xCE) || (rev >= 0xE1 && rev <= 0xE6))
1457
1458static int vi_common_early_init(void *handle)
1459{
1460	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1461
1462	if (adev->flags & AMD_IS_APU) {
1463		adev->smc_rreg = &cz_smc_rreg;
1464		adev->smc_wreg = &cz_smc_wreg;
1465	} else {
1466		adev->smc_rreg = &vi_smc_rreg;
1467		adev->smc_wreg = &vi_smc_wreg;
1468	}
1469	adev->pcie_rreg = &vi_pcie_rreg;
1470	adev->pcie_wreg = &vi_pcie_wreg;
1471	adev->uvd_ctx_rreg = &vi_uvd_ctx_rreg;
1472	adev->uvd_ctx_wreg = &vi_uvd_ctx_wreg;
1473	adev->didt_rreg = &vi_didt_rreg;
1474	adev->didt_wreg = &vi_didt_wreg;
1475	adev->gc_cac_rreg = &vi_gc_cac_rreg;
1476	adev->gc_cac_wreg = &vi_gc_cac_wreg;
1477
1478	adev->asic_funcs = &vi_asic_funcs;
1479
1480	adev->rev_id = vi_get_rev_id(adev);
1481	adev->external_rev_id = 0xFF;
1482	switch (adev->asic_type) {
1483	case CHIP_TOPAZ:
1484		adev->cg_flags = 0;
1485		adev->pg_flags = 0;
1486		adev->external_rev_id = 0x1;
1487		break;
1488	case CHIP_FIJI:
1489		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1490			AMD_CG_SUPPORT_GFX_MGLS |
1491			AMD_CG_SUPPORT_GFX_RLC_LS |
1492			AMD_CG_SUPPORT_GFX_CP_LS |
1493			AMD_CG_SUPPORT_GFX_CGTS |
1494			AMD_CG_SUPPORT_GFX_CGTS_LS |
1495			AMD_CG_SUPPORT_GFX_CGCG |
1496			AMD_CG_SUPPORT_GFX_CGLS |
1497			AMD_CG_SUPPORT_SDMA_MGCG |
1498			AMD_CG_SUPPORT_SDMA_LS |
1499			AMD_CG_SUPPORT_BIF_LS |
1500			AMD_CG_SUPPORT_HDP_MGCG |
1501			AMD_CG_SUPPORT_HDP_LS |
1502			AMD_CG_SUPPORT_ROM_MGCG |
1503			AMD_CG_SUPPORT_MC_MGCG |
1504			AMD_CG_SUPPORT_MC_LS |
1505			AMD_CG_SUPPORT_UVD_MGCG;
1506		adev->pg_flags = 0;
1507		adev->external_rev_id = adev->rev_id + 0x3c;
1508		break;
1509	case CHIP_TONGA:
1510		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1511			AMD_CG_SUPPORT_GFX_CGCG |
1512			AMD_CG_SUPPORT_GFX_CGLS |
1513			AMD_CG_SUPPORT_SDMA_MGCG |
1514			AMD_CG_SUPPORT_SDMA_LS |
1515			AMD_CG_SUPPORT_BIF_LS |
1516			AMD_CG_SUPPORT_HDP_MGCG |
1517			AMD_CG_SUPPORT_HDP_LS |
1518			AMD_CG_SUPPORT_ROM_MGCG |
1519			AMD_CG_SUPPORT_MC_MGCG |
1520			AMD_CG_SUPPORT_MC_LS |
1521			AMD_CG_SUPPORT_DRM_LS |
1522			AMD_CG_SUPPORT_UVD_MGCG;
1523		adev->pg_flags = 0;
1524		adev->external_rev_id = adev->rev_id + 0x14;
1525		break;
1526	case CHIP_POLARIS11:
1527		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1528			AMD_CG_SUPPORT_GFX_RLC_LS |
1529			AMD_CG_SUPPORT_GFX_CP_LS |
1530			AMD_CG_SUPPORT_GFX_CGCG |
1531			AMD_CG_SUPPORT_GFX_CGLS |
1532			AMD_CG_SUPPORT_GFX_3D_CGCG |
1533			AMD_CG_SUPPORT_GFX_3D_CGLS |
1534			AMD_CG_SUPPORT_SDMA_MGCG |
1535			AMD_CG_SUPPORT_SDMA_LS |
1536			AMD_CG_SUPPORT_BIF_MGCG |
1537			AMD_CG_SUPPORT_BIF_LS |
1538			AMD_CG_SUPPORT_HDP_MGCG |
1539			AMD_CG_SUPPORT_HDP_LS |
1540			AMD_CG_SUPPORT_ROM_MGCG |
1541			AMD_CG_SUPPORT_MC_MGCG |
1542			AMD_CG_SUPPORT_MC_LS |
1543			AMD_CG_SUPPORT_DRM_LS |
1544			AMD_CG_SUPPORT_UVD_MGCG |
1545			AMD_CG_SUPPORT_VCE_MGCG;
1546		adev->pg_flags = 0;
1547		adev->external_rev_id = adev->rev_id + 0x5A;
1548		break;
1549	case CHIP_POLARIS10:
1550		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1551			AMD_CG_SUPPORT_GFX_RLC_LS |
1552			AMD_CG_SUPPORT_GFX_CP_LS |
1553			AMD_CG_SUPPORT_GFX_CGCG |
1554			AMD_CG_SUPPORT_GFX_CGLS |
1555			AMD_CG_SUPPORT_GFX_3D_CGCG |
1556			AMD_CG_SUPPORT_GFX_3D_CGLS |
1557			AMD_CG_SUPPORT_SDMA_MGCG |
1558			AMD_CG_SUPPORT_SDMA_LS |
1559			AMD_CG_SUPPORT_BIF_MGCG |
1560			AMD_CG_SUPPORT_BIF_LS |
1561			AMD_CG_SUPPORT_HDP_MGCG |
1562			AMD_CG_SUPPORT_HDP_LS |
1563			AMD_CG_SUPPORT_ROM_MGCG |
1564			AMD_CG_SUPPORT_MC_MGCG |
1565			AMD_CG_SUPPORT_MC_LS |
1566			AMD_CG_SUPPORT_DRM_LS |
1567			AMD_CG_SUPPORT_UVD_MGCG |
1568			AMD_CG_SUPPORT_VCE_MGCG;
1569		adev->pg_flags = 0;
1570		adev->external_rev_id = adev->rev_id + 0x50;
1571		break;
1572	case CHIP_POLARIS12:
1573		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
1574			AMD_CG_SUPPORT_GFX_RLC_LS |
1575			AMD_CG_SUPPORT_GFX_CP_LS |
1576			AMD_CG_SUPPORT_GFX_CGCG |
1577			AMD_CG_SUPPORT_GFX_CGLS |
1578			AMD_CG_SUPPORT_GFX_3D_CGCG |
1579			AMD_CG_SUPPORT_GFX_3D_CGLS |
1580			AMD_CG_SUPPORT_SDMA_MGCG |
1581			AMD_CG_SUPPORT_SDMA_LS |
1582			AMD_CG_SUPPORT_BIF_MGCG |
1583			AMD_CG_SUPPORT_BIF_LS |
1584			AMD_CG_SUPPORT_HDP_MGCG |
1585			AMD_CG_SUPPORT_HDP_LS |
1586			AMD_CG_SUPPORT_ROM_MGCG |
1587			AMD_CG_SUPPORT_MC_MGCG |
1588			AMD_CG_SUPPORT_MC_LS |
1589			AMD_CG_SUPPORT_DRM_LS |
1590			AMD_CG_SUPPORT_UVD_MGCG |
1591			AMD_CG_SUPPORT_VCE_MGCG;
1592		adev->pg_flags = 0;
1593		adev->external_rev_id = adev->rev_id + 0x64;
1594		break;
1595	case CHIP_VEGAM:
1596		adev->cg_flags = 0;
1597			/*AMD_CG_SUPPORT_GFX_MGCG |
1598			AMD_CG_SUPPORT_GFX_RLC_LS |
1599			AMD_CG_SUPPORT_GFX_CP_LS |
1600			AMD_CG_SUPPORT_GFX_CGCG |
1601			AMD_CG_SUPPORT_GFX_CGLS |
1602			AMD_CG_SUPPORT_GFX_3D_CGCG |
1603			AMD_CG_SUPPORT_GFX_3D_CGLS |
1604			AMD_CG_SUPPORT_SDMA_MGCG |
1605			AMD_CG_SUPPORT_SDMA_LS |
1606			AMD_CG_SUPPORT_BIF_MGCG |
1607			AMD_CG_SUPPORT_BIF_LS |
1608			AMD_CG_SUPPORT_HDP_MGCG |
1609			AMD_CG_SUPPORT_HDP_LS |
1610			AMD_CG_SUPPORT_ROM_MGCG |
1611			AMD_CG_SUPPORT_MC_MGCG |
1612			AMD_CG_SUPPORT_MC_LS |
1613			AMD_CG_SUPPORT_DRM_LS |
1614			AMD_CG_SUPPORT_UVD_MGCG |
1615			AMD_CG_SUPPORT_VCE_MGCG;*/
1616		adev->pg_flags = 0;
1617		adev->external_rev_id = adev->rev_id + 0x6E;
1618		break;
1619	case CHIP_CARRIZO:
1620		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
1621			AMD_CG_SUPPORT_GFX_MGCG |
1622			AMD_CG_SUPPORT_GFX_MGLS |
1623			AMD_CG_SUPPORT_GFX_RLC_LS |
1624			AMD_CG_SUPPORT_GFX_CP_LS |
1625			AMD_CG_SUPPORT_GFX_CGTS |
1626			AMD_CG_SUPPORT_GFX_CGTS_LS |
1627			AMD_CG_SUPPORT_GFX_CGCG |
1628			AMD_CG_SUPPORT_GFX_CGLS |
1629			AMD_CG_SUPPORT_BIF_LS |
1630			AMD_CG_SUPPORT_HDP_MGCG |
1631			AMD_CG_SUPPORT_HDP_LS |
1632			AMD_CG_SUPPORT_SDMA_MGCG |
1633			AMD_CG_SUPPORT_SDMA_LS |
1634			AMD_CG_SUPPORT_VCE_MGCG;
1635		/* rev0 hardware requires workarounds to support PG */
1636		adev->pg_flags = 0;
1637		if (adev->rev_id != 0x00 || CZ_REV_BRISTOL(adev->pdev->revision)) {
1638			adev->pg_flags |= AMD_PG_SUPPORT_GFX_SMG |
1639				AMD_PG_SUPPORT_GFX_PIPELINE |
1640				AMD_PG_SUPPORT_CP |
1641				AMD_PG_SUPPORT_UVD |
1642				AMD_PG_SUPPORT_VCE;
1643		}
1644		adev->external_rev_id = adev->rev_id + 0x1;
1645		break;
1646	case CHIP_STONEY:
1647		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
1648			AMD_CG_SUPPORT_GFX_MGCG |
1649			AMD_CG_SUPPORT_GFX_MGLS |
1650			AMD_CG_SUPPORT_GFX_RLC_LS |
1651			AMD_CG_SUPPORT_GFX_CP_LS |
1652			AMD_CG_SUPPORT_GFX_CGTS |
1653			AMD_CG_SUPPORT_GFX_CGTS_LS |
1654			AMD_CG_SUPPORT_GFX_CGLS |
1655			AMD_CG_SUPPORT_BIF_LS |
1656			AMD_CG_SUPPORT_HDP_MGCG |
1657			AMD_CG_SUPPORT_HDP_LS |
1658			AMD_CG_SUPPORT_SDMA_MGCG |
1659			AMD_CG_SUPPORT_SDMA_LS |
1660			AMD_CG_SUPPORT_VCE_MGCG;
1661		adev->pg_flags = AMD_PG_SUPPORT_GFX_PG |
1662			AMD_PG_SUPPORT_GFX_SMG |
1663			AMD_PG_SUPPORT_GFX_PIPELINE |
1664			AMD_PG_SUPPORT_CP |
1665			AMD_PG_SUPPORT_UVD |
1666			AMD_PG_SUPPORT_VCE;
1667		adev->external_rev_id = adev->rev_id + 0x61;
1668		break;
1669	default:
1670		/* FIXME: not supported yet */
1671		return -EINVAL;
1672	}
1673
1674	if (amdgpu_sriov_vf(adev)) {
1675		amdgpu_virt_init_setting(adev);
1676		xgpu_vi_mailbox_set_irq_funcs(adev);
1677	}
1678
1679	return 0;
1680}
1681
1682static int vi_common_late_init(void *handle)
1683{
1684	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1685
1686	if (amdgpu_sriov_vf(adev))
1687		xgpu_vi_mailbox_get_irq(adev);
1688
1689	return 0;
1690}
1691
1692static int vi_common_sw_init(void *handle)
1693{
1694	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1695
1696	if (amdgpu_sriov_vf(adev))
1697		xgpu_vi_mailbox_add_irq_id(adev);
1698
1699	return 0;
1700}
1701
1702static int vi_common_sw_fini(void *handle)
1703{
1704	return 0;
1705}
1706
1707static int vi_common_hw_init(void *handle)
1708{
1709	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1710
1711	/* move the golden regs per IP block */
1712	vi_init_golden_registers(adev);
 
 
1713	/* enable aspm */
1714	vi_program_aspm(adev);
1715	/* enable the doorbell aperture */
1716	vi_enable_doorbell_aperture(adev, true);
1717
1718	return 0;
1719}
1720
1721static int vi_common_hw_fini(void *handle)
1722{
1723	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1724
1725	/* enable the doorbell aperture */
1726	vi_enable_doorbell_aperture(adev, false);
1727
1728	if (amdgpu_sriov_vf(adev))
1729		xgpu_vi_mailbox_put_irq(adev);
1730
1731	return 0;
1732}
1733
1734static int vi_common_suspend(void *handle)
1735{
1736	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1737
1738	return vi_common_hw_fini(adev);
1739}
1740
1741static int vi_common_resume(void *handle)
1742{
1743	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1744
1745	return vi_common_hw_init(adev);
1746}
1747
1748static bool vi_common_is_idle(void *handle)
1749{
1750	return true;
1751}
1752
1753static int vi_common_wait_for_idle(void *handle)
1754{
1755	return 0;
1756}
1757
1758static int vi_common_soft_reset(void *handle)
1759{
1760	return 0;
1761}
1762
1763static void vi_update_bif_medium_grain_light_sleep(struct amdgpu_device *adev,
1764						   bool enable)
1765{
1766	uint32_t temp, data;
1767
1768	temp = data = RREG32_PCIE(ixPCIE_CNTL2);
1769
1770	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS))
1771		data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1772				PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1773				PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1774	else
1775		data &= ~(PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1776				PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1777				PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
1778
1779	if (temp != data)
1780		WREG32_PCIE(ixPCIE_CNTL2, data);
1781}
1782
1783static void vi_update_hdp_medium_grain_clock_gating(struct amdgpu_device *adev,
1784						    bool enable)
1785{
1786	uint32_t temp, data;
1787
1788	temp = data = RREG32(mmHDP_HOST_PATH_CNTL);
1789
1790	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
1791		data &= ~HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK;
1792	else
1793		data |= HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK;
1794
1795	if (temp != data)
1796		WREG32(mmHDP_HOST_PATH_CNTL, data);
1797}
1798
1799static void vi_update_hdp_light_sleep(struct amdgpu_device *adev,
1800				      bool enable)
1801{
1802	uint32_t temp, data;
1803
1804	temp = data = RREG32(mmHDP_MEM_POWER_LS);
1805
1806	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
1807		data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK;
1808	else
1809		data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK;
1810
1811	if (temp != data)
1812		WREG32(mmHDP_MEM_POWER_LS, data);
1813}
1814
1815static void vi_update_drm_light_sleep(struct amdgpu_device *adev,
1816				      bool enable)
1817{
1818	uint32_t temp, data;
1819
1820	temp = data = RREG32(0x157a);
1821
1822	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS))
1823		data |= 1;
1824	else
1825		data &= ~1;
1826
1827	if (temp != data)
1828		WREG32(0x157a, data);
1829}
1830
1831
1832static void vi_update_rom_medium_grain_clock_gating(struct amdgpu_device *adev,
1833						    bool enable)
1834{
1835	uint32_t temp, data;
1836
1837	temp = data = RREG32_SMC(ixCGTT_ROM_CLK_CTRL0);
1838
1839	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG))
1840		data &= ~(CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK |
1841				CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK);
1842	else
1843		data |= CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK |
1844				CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK;
1845
1846	if (temp != data)
1847		WREG32_SMC(ixCGTT_ROM_CLK_CTRL0, data);
1848}
1849
1850static int vi_common_set_clockgating_state_by_smu(void *handle,
1851					   enum amd_clockgating_state state)
1852{
1853	uint32_t msg_id, pp_state = 0;
1854	uint32_t pp_support_state = 0;
1855	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1856
1857	if (adev->cg_flags & (AMD_CG_SUPPORT_MC_LS | AMD_CG_SUPPORT_MC_MGCG)) {
1858		if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) {
1859			pp_support_state = PP_STATE_SUPPORT_LS;
1860			pp_state = PP_STATE_LS;
1861		}
1862		if (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG) {
1863			pp_support_state |= PP_STATE_SUPPORT_CG;
1864			pp_state |= PP_STATE_CG;
1865		}
1866		if (state == AMD_CG_STATE_UNGATE)
1867			pp_state = 0;
1868		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1869			       PP_BLOCK_SYS_MC,
1870			       pp_support_state,
1871			       pp_state);
1872		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1873	}
1874
1875	if (adev->cg_flags & (AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_SDMA_MGCG)) {
1876		if (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS) {
1877			pp_support_state = PP_STATE_SUPPORT_LS;
1878			pp_state = PP_STATE_LS;
1879		}
1880		if (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG) {
1881			pp_support_state |= PP_STATE_SUPPORT_CG;
1882			pp_state |= PP_STATE_CG;
1883		}
1884		if (state == AMD_CG_STATE_UNGATE)
1885			pp_state = 0;
1886		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1887			       PP_BLOCK_SYS_SDMA,
1888			       pp_support_state,
1889			       pp_state);
1890		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1891	}
1892
1893	if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_MGCG)) {
1894		if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) {
1895			pp_support_state = PP_STATE_SUPPORT_LS;
1896			pp_state = PP_STATE_LS;
1897		}
1898		if (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG) {
1899			pp_support_state |= PP_STATE_SUPPORT_CG;
1900			pp_state |= PP_STATE_CG;
1901		}
1902		if (state == AMD_CG_STATE_UNGATE)
1903			pp_state = 0;
1904		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1905			       PP_BLOCK_SYS_HDP,
1906			       pp_support_state,
1907			       pp_state);
1908		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1909	}
1910
1911
1912	if (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS) {
1913		if (state == AMD_CG_STATE_UNGATE)
1914			pp_state = 0;
1915		else
1916			pp_state = PP_STATE_LS;
1917
1918		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1919			       PP_BLOCK_SYS_BIF,
1920			       PP_STATE_SUPPORT_LS,
1921			        pp_state);
1922		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1923	}
1924	if (adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG) {
1925		if (state == AMD_CG_STATE_UNGATE)
1926			pp_state = 0;
1927		else
1928			pp_state = PP_STATE_CG;
1929
1930		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1931			       PP_BLOCK_SYS_BIF,
1932			       PP_STATE_SUPPORT_CG,
1933			       pp_state);
1934		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1935	}
1936
1937	if (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS) {
1938
1939		if (state == AMD_CG_STATE_UNGATE)
1940			pp_state = 0;
1941		else
1942			pp_state = PP_STATE_LS;
1943
1944		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1945			       PP_BLOCK_SYS_DRM,
1946			       PP_STATE_SUPPORT_LS,
1947			       pp_state);
1948		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1949	}
1950
1951	if (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG) {
1952
1953		if (state == AMD_CG_STATE_UNGATE)
1954			pp_state = 0;
1955		else
1956			pp_state = PP_STATE_CG;
1957
1958		msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
1959			       PP_BLOCK_SYS_ROM,
1960			       PP_STATE_SUPPORT_CG,
1961			       pp_state);
1962		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
1963	}
1964	return 0;
1965}
1966
1967static int vi_common_set_clockgating_state(void *handle,
1968					   enum amd_clockgating_state state)
1969{
1970	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1971
1972	if (amdgpu_sriov_vf(adev))
1973		return 0;
1974
1975	switch (adev->asic_type) {
1976	case CHIP_FIJI:
1977		vi_update_bif_medium_grain_light_sleep(adev,
1978				state == AMD_CG_STATE_GATE);
1979		vi_update_hdp_medium_grain_clock_gating(adev,
1980				state == AMD_CG_STATE_GATE);
1981		vi_update_hdp_light_sleep(adev,
1982				state == AMD_CG_STATE_GATE);
1983		vi_update_rom_medium_grain_clock_gating(adev,
1984				state == AMD_CG_STATE_GATE);
1985		break;
1986	case CHIP_CARRIZO:
1987	case CHIP_STONEY:
1988		vi_update_bif_medium_grain_light_sleep(adev,
1989				state == AMD_CG_STATE_GATE);
1990		vi_update_hdp_medium_grain_clock_gating(adev,
1991				state == AMD_CG_STATE_GATE);
1992		vi_update_hdp_light_sleep(adev,
1993				state == AMD_CG_STATE_GATE);
1994		vi_update_drm_light_sleep(adev,
1995				state == AMD_CG_STATE_GATE);
1996		break;
1997	case CHIP_TONGA:
1998	case CHIP_POLARIS10:
1999	case CHIP_POLARIS11:
2000	case CHIP_POLARIS12:
2001	case CHIP_VEGAM:
2002		vi_common_set_clockgating_state_by_smu(adev, state);
2003		break;
2004	default:
2005		break;
2006	}
2007	return 0;
2008}
2009
2010static int vi_common_set_powergating_state(void *handle,
2011					    enum amd_powergating_state state)
2012{
2013	return 0;
2014}
2015
2016static void vi_common_get_clockgating_state(void *handle, u64 *flags)
2017{
2018	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2019	int data;
2020
2021	if (amdgpu_sriov_vf(adev))
2022		*flags = 0;
2023
2024	/* AMD_CG_SUPPORT_BIF_LS */
2025	data = RREG32_PCIE(ixPCIE_CNTL2);
2026	if (data & PCIE_CNTL2__SLV_MEM_LS_EN_MASK)
2027		*flags |= AMD_CG_SUPPORT_BIF_LS;
2028
2029	/* AMD_CG_SUPPORT_HDP_LS */
2030	data = RREG32(mmHDP_MEM_POWER_LS);
2031	if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK)
2032		*flags |= AMD_CG_SUPPORT_HDP_LS;
2033
2034	/* AMD_CG_SUPPORT_HDP_MGCG */
2035	data = RREG32(mmHDP_HOST_PATH_CNTL);
2036	if (!(data & HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK))
2037		*flags |= AMD_CG_SUPPORT_HDP_MGCG;
2038
2039	/* AMD_CG_SUPPORT_ROM_MGCG */
2040	data = RREG32_SMC(ixCGTT_ROM_CLK_CTRL0);
2041	if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK))
2042		*flags |= AMD_CG_SUPPORT_ROM_MGCG;
2043}
2044
2045static const struct amd_ip_funcs vi_common_ip_funcs = {
2046	.name = "vi_common",
2047	.early_init = vi_common_early_init,
2048	.late_init = vi_common_late_init,
2049	.sw_init = vi_common_sw_init,
2050	.sw_fini = vi_common_sw_fini,
2051	.hw_init = vi_common_hw_init,
2052	.hw_fini = vi_common_hw_fini,
2053	.suspend = vi_common_suspend,
2054	.resume = vi_common_resume,
2055	.is_idle = vi_common_is_idle,
2056	.wait_for_idle = vi_common_wait_for_idle,
2057	.soft_reset = vi_common_soft_reset,
2058	.set_clockgating_state = vi_common_set_clockgating_state,
2059	.set_powergating_state = vi_common_set_powergating_state,
2060	.get_clockgating_state = vi_common_get_clockgating_state,
2061};
2062
2063static const struct amdgpu_ip_block_version vi_common_ip_block =
2064{
2065	.type = AMD_IP_BLOCK_TYPE_COMMON,
2066	.major = 1,
2067	.minor = 0,
2068	.rev = 0,
2069	.funcs = &vi_common_ip_funcs,
2070};
2071
2072void vi_set_virt_ops(struct amdgpu_device *adev)
2073{
2074	adev->virt.ops = &xgpu_vi_virt_ops;
2075}
2076
2077int vi_set_ip_blocks(struct amdgpu_device *adev)
2078{
2079	amdgpu_device_set_sriov_virtual_display(adev);
2080
2081	switch (adev->asic_type) {
2082	case CHIP_TOPAZ:
2083		/* topaz has no DCE, UVD, VCE */
2084		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2085		amdgpu_device_ip_block_add(adev, &gmc_v7_4_ip_block);
2086		amdgpu_device_ip_block_add(adev, &iceland_ih_ip_block);
2087		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
2088		amdgpu_device_ip_block_add(adev, &sdma_v2_4_ip_block);
2089		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2090		if (adev->enable_virtual_display)
2091			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2092		break;
2093	case CHIP_FIJI:
2094		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2095		amdgpu_device_ip_block_add(adev, &gmc_v8_5_ip_block);
2096		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
2097		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
2098		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
2099		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2100		if (adev->enable_virtual_display)
2101			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2102#if defined(CONFIG_DRM_AMD_DC)
2103		else if (amdgpu_device_has_dc_support(adev))
2104			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2105#endif
2106		else
2107			amdgpu_device_ip_block_add(adev, &dce_v10_1_ip_block);
2108		if (!amdgpu_sriov_vf(adev)) {
2109			amdgpu_device_ip_block_add(adev, &uvd_v6_0_ip_block);
2110			amdgpu_device_ip_block_add(adev, &vce_v3_0_ip_block);
2111		}
2112		break;
2113	case CHIP_TONGA:
2114		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2115		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
2116		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
2117		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
2118		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
2119		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2120		if (adev->enable_virtual_display)
2121			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2122#if defined(CONFIG_DRM_AMD_DC)
2123		else if (amdgpu_device_has_dc_support(adev))
2124			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2125#endif
2126		else
2127			amdgpu_device_ip_block_add(adev, &dce_v10_0_ip_block);
2128		if (!amdgpu_sriov_vf(adev)) {
2129			amdgpu_device_ip_block_add(adev, &uvd_v5_0_ip_block);
2130			amdgpu_device_ip_block_add(adev, &vce_v3_0_ip_block);
2131		}
2132		break;
2133	case CHIP_POLARIS10:
2134	case CHIP_POLARIS11:
2135	case CHIP_POLARIS12:
2136	case CHIP_VEGAM:
2137		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2138		amdgpu_device_ip_block_add(adev, &gmc_v8_1_ip_block);
2139		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
2140		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
2141		amdgpu_device_ip_block_add(adev, &sdma_v3_1_ip_block);
2142		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2143		if (adev->enable_virtual_display)
2144			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2145#if defined(CONFIG_DRM_AMD_DC)
2146		else if (amdgpu_device_has_dc_support(adev))
2147			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2148#endif
2149		else
2150			amdgpu_device_ip_block_add(adev, &dce_v11_2_ip_block);
2151		amdgpu_device_ip_block_add(adev, &uvd_v6_3_ip_block);
2152		amdgpu_device_ip_block_add(adev, &vce_v3_4_ip_block);
2153		break;
2154	case CHIP_CARRIZO:
2155		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2156		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
2157		amdgpu_device_ip_block_add(adev, &cz_ih_ip_block);
2158		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
2159		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
2160		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2161		if (adev->enable_virtual_display)
2162			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2163#if defined(CONFIG_DRM_AMD_DC)
2164		else if (amdgpu_device_has_dc_support(adev))
2165			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2166#endif
2167		else
2168			amdgpu_device_ip_block_add(adev, &dce_v11_0_ip_block);
2169		amdgpu_device_ip_block_add(adev, &uvd_v6_0_ip_block);
2170		amdgpu_device_ip_block_add(adev, &vce_v3_1_ip_block);
2171#if defined(CONFIG_DRM_AMD_ACP)
2172		amdgpu_device_ip_block_add(adev, &acp_ip_block);
2173#endif
2174		break;
2175	case CHIP_STONEY:
2176		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
2177		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
2178		amdgpu_device_ip_block_add(adev, &cz_ih_ip_block);
2179		amdgpu_device_ip_block_add(adev, &gfx_v8_1_ip_block);
2180		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
2181		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2182		if (adev->enable_virtual_display)
2183			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2184#if defined(CONFIG_DRM_AMD_DC)
2185		else if (amdgpu_device_has_dc_support(adev))
2186			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2187#endif
2188		else
2189			amdgpu_device_ip_block_add(adev, &dce_v11_0_ip_block);
2190		amdgpu_device_ip_block_add(adev, &uvd_v6_2_ip_block);
2191		amdgpu_device_ip_block_add(adev, &vce_v3_4_ip_block);
2192#if defined(CONFIG_DRM_AMD_ACP)
2193		amdgpu_device_ip_block_add(adev, &acp_ip_block);
2194#endif
2195		break;
2196	default:
2197		/* FIXME: not supported yet */
2198		return -EINVAL;
2199	}
2200
2201	return 0;
2202}
2203
2204void legacy_doorbell_index_init(struct amdgpu_device *adev)
2205{
2206	adev->doorbell_index.kiq = AMDGPU_DOORBELL_KIQ;
2207	adev->doorbell_index.mec_ring0 = AMDGPU_DOORBELL_MEC_RING0;
2208	adev->doorbell_index.mec_ring1 = AMDGPU_DOORBELL_MEC_RING1;
2209	adev->doorbell_index.mec_ring2 = AMDGPU_DOORBELL_MEC_RING2;
2210	adev->doorbell_index.mec_ring3 = AMDGPU_DOORBELL_MEC_RING3;
2211	adev->doorbell_index.mec_ring4 = AMDGPU_DOORBELL_MEC_RING4;
2212	adev->doorbell_index.mec_ring5 = AMDGPU_DOORBELL_MEC_RING5;
2213	adev->doorbell_index.mec_ring6 = AMDGPU_DOORBELL_MEC_RING6;
2214	adev->doorbell_index.mec_ring7 = AMDGPU_DOORBELL_MEC_RING7;
2215	adev->doorbell_index.gfx_ring0 = AMDGPU_DOORBELL_GFX_RING0;
2216	adev->doorbell_index.sdma_engine[0] = AMDGPU_DOORBELL_sDMA_ENGINE0;
2217	adev->doorbell_index.sdma_engine[1] = AMDGPU_DOORBELL_sDMA_ENGINE1;
2218	adev->doorbell_index.ih = AMDGPU_DOORBELL_IH;
2219	adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_MAX_ASSIGNMENT;
2220}