1/*
   2 * Copyright 2015 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_atombios.h"
  31#include "amdgpu_ih.h"
  32#include "amdgpu_uvd.h"
  33#include "amdgpu_vce.h"
  34#include "atom.h"
  35#include "amd_pcie.h"
  36#include "si_dpm.h"
  37#include "sid.h"
  38#include "si_ih.h"
  39#include "gfx_v6_0.h"
  40#include "gmc_v6_0.h"
  41#include "si_dma.h"
  42#include "dce_v6_0.h"
  43#include "si.h"
  44#include "dce_virtual.h"
  45#include "gca/gfx_6_0_d.h"
  46#include "oss/oss_1_0_d.h"
  47#include "gmc/gmc_6_0_d.h"
  48#include "dce/dce_6_0_d.h"
  49#include "uvd/uvd_4_0_d.h"
  50#include "bif/bif_3_0_d.h"
  51#include "bif/bif_3_0_sh_mask.h"
  52
  53static const u32 tahiti_golden_registers[] =
  54{
  55	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  56	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  57	mmDB_DEBUG, 0xffffffff, 0x00000000,
  58	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  59	mmDB_DEBUG3, 0x0002021c, 0x00020200,
  60	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  61	0x340c, 0x000000c0, 0x00800040,
  62	0x360c, 0x000000c0, 0x00800040,
  63	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  64	mmFBC_MISC, 0x00200000, 0x50100000,
  65	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  66	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  67	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  68	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  69	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  70	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  71	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  72	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  73	0x000c, 0xffffffff, 0x0040,
  74	0x000d, 0x00000040, 0x00004040,
  75	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  76	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  77	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  78	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  79	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  80	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  81	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  82	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  83	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  84	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  85	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  86	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  87	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  88	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  89	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  90	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  91};
  92
  93static const u32 tahiti_golden_registers2[] =
  94{
  95	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  96};
  97
  98static const u32 tahiti_golden_rlc_registers[] =
  99{
 100	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 101	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 102	0x311f, 0xffffffff, 0x10104040,
 103	0x3122, 0xffffffff, 0x0100000a,
 104	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 105	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 106	mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 107};
 108
 109static const u32 pitcairn_golden_registers[] =
 110{
 111	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 112	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 113	mmDB_DEBUG, 0xffffffff, 0x00000000,
 114	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 115	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 116	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 117	0x340c, 0x000300c0, 0x00800040,
 118	0x360c, 0x000300c0, 0x00800040,
 119	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 120	mmFBC_MISC, 0x00200000, 0x50100000,
 121	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 122	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 123	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 124	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 125	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 126	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 127	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 128	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 129	0x000c, 0xffffffff, 0x0040,
 130	0x000d, 0x00000040, 0x00004040,
 131	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 132	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 133	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 134	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 135	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 136	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 137	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 138	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 139	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 140	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 141	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 142	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 143};
 144
 145static const u32 pitcairn_golden_rlc_registers[] =
 146{
 147	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 148	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 149	0x311f, 0xffffffff, 0x10102020,
 150	0x3122, 0xffffffff, 0x01000020,
 151	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 152	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 153};
 154
 155static const u32 verde_pg_init[] =
 156{
 157	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 158	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 159	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 160	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 161	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 162	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 163	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 164	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 165	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 166	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 167	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 168	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 169	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 170	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 171	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 172	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 173	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 174	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 175	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 176	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 177	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 178	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 179	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 180	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 181	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 182	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 183	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 184	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 186	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 187	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 188	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 189	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 190	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 191	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 193	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 194	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 195	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 196	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 198	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 199	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 200	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 201	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 202	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 203	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 204	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 205	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 206	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 207	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 208	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 209	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 210	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 211	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 212	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 213	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 214	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 215	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 216	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 217	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 218	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 219	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 220	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 221	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 222	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 223	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 224	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 225	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 226	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 227	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 228	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 229	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 230	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 231	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 232	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 233	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 234	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 235	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 236	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 237	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 238	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 239	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 240	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 241	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 242	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 243	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 244	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 245	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 246	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 247	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 248	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 249	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 250	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 251	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 252	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 253	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 254	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 255	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 256	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 257	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 258	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 259	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 260	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 261	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 262	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 263	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 264	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 265	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 266	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 267	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 268	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 269	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 270	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 271	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 272	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 273	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 274	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 275	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 276	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 277	mmGMCON_MISC2, 0xfc00, 0x2000,
 278	mmGMCON_MISC3, 0xffffffff, 0xfc0,
 279	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 280};
 281
 282static const u32 verde_golden_rlc_registers[] =
 283{
 284	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 285	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 286	0x311f, 0xffffffff, 0x10808020,
 287	0x3122, 0xffffffff, 0x00800008,
 288	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 289	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 290};
 291
 292static const u32 verde_golden_registers[] =
 293{
 294	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 295	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 296	mmDB_DEBUG, 0xffffffff, 0x00000000,
 297	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 298	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 299	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 300	0x340c, 0x000300c0, 0x00800040,
 301	0x360c, 0x000300c0, 0x00800040,
 302	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 303	mmFBC_MISC, 0x00200000, 0x50100000,
 304	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 305	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 306	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 307	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 308	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 309	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 310	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 311	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 312	0x000c, 0xffffffff, 0x0040,
 313	0x000d, 0x00000040, 0x00004040,
 314	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 315	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 316	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 317	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 318	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 319	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 320	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 321	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 322	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 323	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 324	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 325	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 326	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 327	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 328};
 329
 330static const u32 oland_golden_registers[] =
 331{
 332	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 333	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 334	mmDB_DEBUG, 0xffffffff, 0x00000000,
 335	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 336	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 337	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 338	0x340c, 0x000300c0, 0x00800040,
 339	0x360c, 0x000300c0, 0x00800040,
 340	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 341	mmFBC_MISC, 0x00200000, 0x50100000,
 342	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 343	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 344	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 345	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 346	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 347	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 348	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 349	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 350	0x000c, 0xffffffff, 0x0040,
 351	0x000d, 0x00000040, 0x00004040,
 352	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 353	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 354	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 355	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 356	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 357	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 358	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 359	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 360	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 361	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 362	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 363	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 364
 365};
 366
 367static const u32 oland_golden_rlc_registers[] =
 368{
 369	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 370	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 371	0x311f, 0xffffffff, 0x10104040,
 372	0x3122, 0xffffffff, 0x0100000a,
 373	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 374	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 375};
 376
 377static const u32 hainan_golden_registers[] =
 378{
 379	0x17bc, 0x00000030, 0x00000011,
 380	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 381	mmDB_DEBUG, 0xffffffff, 0x00000000,
 382	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 383	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 384	0x031e, 0x00000080, 0x00000000,
 385	0x3430, 0xff000fff, 0x00000100,
 386	0x340c, 0x000300c0, 0x00800040,
 387	0x3630, 0xff000fff, 0x00000100,
 388	0x360c, 0x000300c0, 0x00800040,
 389	0x16ec, 0x000000f0, 0x00000070,
 390	0x16f0, 0x00200000, 0x50100000,
 391	0x1c0c, 0x31000311, 0x00000011,
 392	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 393	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 394	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 395	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 396	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 397	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 398	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 399	0x000c, 0xffffffff, 0x0040,
 400	0x000d, 0x00000040, 0x00004040,
 401	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 402	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 403	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 404	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 405	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 406	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 407	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 408	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 409	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 410	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 411	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 412	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 413};
 414
 415static const u32 hainan_golden_registers2[] =
 416{
 417	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 418};
 419
 420static const u32 tahiti_mgcg_cgcg_init[] =
 421{
 422	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 423	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 424	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 425	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 426	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 427	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 428	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 429	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 430	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 431	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 432	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 433	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 434	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 435	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 436	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 437	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 438	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 439	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 440	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 441	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 442	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 443	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 444	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 445	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 446	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 447	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 448	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 449	0x2458, 0xffffffff, 0x00010000,
 450	0x2459, 0xffffffff, 0x00030002,
 451	0x245a, 0xffffffff, 0x00040007,
 452	0x245b, 0xffffffff, 0x00060005,
 453	0x245c, 0xffffffff, 0x00090008,
 454	0x245d, 0xffffffff, 0x00020001,
 455	0x245e, 0xffffffff, 0x00040003,
 456	0x245f, 0xffffffff, 0x00000007,
 457	0x2460, 0xffffffff, 0x00060005,
 458	0x2461, 0xffffffff, 0x00090008,
 459	0x2462, 0xffffffff, 0x00030002,
 460	0x2463, 0xffffffff, 0x00050004,
 461	0x2464, 0xffffffff, 0x00000008,
 462	0x2465, 0xffffffff, 0x00070006,
 463	0x2466, 0xffffffff, 0x000a0009,
 464	0x2467, 0xffffffff, 0x00040003,
 465	0x2468, 0xffffffff, 0x00060005,
 466	0x2469, 0xffffffff, 0x00000009,
 467	0x246a, 0xffffffff, 0x00080007,
 468	0x246b, 0xffffffff, 0x000b000a,
 469	0x246c, 0xffffffff, 0x00050004,
 470	0x246d, 0xffffffff, 0x00070006,
 471	0x246e, 0xffffffff, 0x0008000b,
 472	0x246f, 0xffffffff, 0x000a0009,
 473	0x2470, 0xffffffff, 0x000d000c,
 474	0x2471, 0xffffffff, 0x00060005,
 475	0x2472, 0xffffffff, 0x00080007,
 476	0x2473, 0xffffffff, 0x0000000b,
 477	0x2474, 0xffffffff, 0x000a0009,
 478	0x2475, 0xffffffff, 0x000d000c,
 479	0x2476, 0xffffffff, 0x00070006,
 480	0x2477, 0xffffffff, 0x00090008,
 481	0x2478, 0xffffffff, 0x0000000c,
 482	0x2479, 0xffffffff, 0x000b000a,
 483	0x247a, 0xffffffff, 0x000e000d,
 484	0x247b, 0xffffffff, 0x00080007,
 485	0x247c, 0xffffffff, 0x000a0009,
 486	0x247d, 0xffffffff, 0x0000000d,
 487	0x247e, 0xffffffff, 0x000c000b,
 488	0x247f, 0xffffffff, 0x000f000e,
 489	0x2480, 0xffffffff, 0x00090008,
 490	0x2481, 0xffffffff, 0x000b000a,
 491	0x2482, 0xffffffff, 0x000c000f,
 492	0x2483, 0xffffffff, 0x000e000d,
 493	0x2484, 0xffffffff, 0x00110010,
 494	0x2485, 0xffffffff, 0x000a0009,
 495	0x2486, 0xffffffff, 0x000c000b,
 496	0x2487, 0xffffffff, 0x0000000f,
 497	0x2488, 0xffffffff, 0x000e000d,
 498	0x2489, 0xffffffff, 0x00110010,
 499	0x248a, 0xffffffff, 0x000b000a,
 500	0x248b, 0xffffffff, 0x000d000c,
 501	0x248c, 0xffffffff, 0x00000010,
 502	0x248d, 0xffffffff, 0x000f000e,
 503	0x248e, 0xffffffff, 0x00120011,
 504	0x248f, 0xffffffff, 0x000c000b,
 505	0x2490, 0xffffffff, 0x000e000d,
 506	0x2491, 0xffffffff, 0x00000011,
 507	0x2492, 0xffffffff, 0x0010000f,
 508	0x2493, 0xffffffff, 0x00130012,
 509	0x2494, 0xffffffff, 0x000d000c,
 510	0x2495, 0xffffffff, 0x000f000e,
 511	0x2496, 0xffffffff, 0x00100013,
 512	0x2497, 0xffffffff, 0x00120011,
 513	0x2498, 0xffffffff, 0x00150014,
 514	0x2499, 0xffffffff, 0x000e000d,
 515	0x249a, 0xffffffff, 0x0010000f,
 516	0x249b, 0xffffffff, 0x00000013,
 517	0x249c, 0xffffffff, 0x00120011,
 518	0x249d, 0xffffffff, 0x00150014,
 519	0x249e, 0xffffffff, 0x000f000e,
 520	0x249f, 0xffffffff, 0x00110010,
 521	0x24a0, 0xffffffff, 0x00000014,
 522	0x24a1, 0xffffffff, 0x00130012,
 523	0x24a2, 0xffffffff, 0x00160015,
 524	0x24a3, 0xffffffff, 0x0010000f,
 525	0x24a4, 0xffffffff, 0x00120011,
 526	0x24a5, 0xffffffff, 0x00000015,
 527	0x24a6, 0xffffffff, 0x00140013,
 528	0x24a7, 0xffffffff, 0x00170016,
 529	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 530	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 531	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 532	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 533	0x000c, 0xffffffff, 0x0000001c,
 534	0x000d, 0x000f0000, 0x000f0000,
 535	0x0583, 0xffffffff, 0x00000100,
 536	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 537	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 538	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 539	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 540	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 541	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 542	0x157a, 0x00000001, 0x00000001,
 543	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 544	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 545	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 546	0x3430, 0xfffffff0, 0x00000100,
 547	0x3630, 0xfffffff0, 0x00000100,
 548};
 549static const u32 pitcairn_mgcg_cgcg_init[] =
 550{
 551	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 552	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 553	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 554	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 555	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 556	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 557	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 558	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 559	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 560	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 561	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 562	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 563	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 564	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 565	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 566	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 567	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 568	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 569	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 570	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 571	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 572	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 573	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 574	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 575	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 576	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 577	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 578	0x2458, 0xffffffff, 0x00010000,
 579	0x2459, 0xffffffff, 0x00030002,
 580	0x245a, 0xffffffff, 0x00040007,
 581	0x245b, 0xffffffff, 0x00060005,
 582	0x245c, 0xffffffff, 0x00090008,
 583	0x245d, 0xffffffff, 0x00020001,
 584	0x245e, 0xffffffff, 0x00040003,
 585	0x245f, 0xffffffff, 0x00000007,
 586	0x2460, 0xffffffff, 0x00060005,
 587	0x2461, 0xffffffff, 0x00090008,
 588	0x2462, 0xffffffff, 0x00030002,
 589	0x2463, 0xffffffff, 0x00050004,
 590	0x2464, 0xffffffff, 0x00000008,
 591	0x2465, 0xffffffff, 0x00070006,
 592	0x2466, 0xffffffff, 0x000a0009,
 593	0x2467, 0xffffffff, 0x00040003,
 594	0x2468, 0xffffffff, 0x00060005,
 595	0x2469, 0xffffffff, 0x00000009,
 596	0x246a, 0xffffffff, 0x00080007,
 597	0x246b, 0xffffffff, 0x000b000a,
 598	0x246c, 0xffffffff, 0x00050004,
 599	0x246d, 0xffffffff, 0x00070006,
 600	0x246e, 0xffffffff, 0x0008000b,
 601	0x246f, 0xffffffff, 0x000a0009,
 602	0x2470, 0xffffffff, 0x000d000c,
 603	0x2480, 0xffffffff, 0x00090008,
 604	0x2481, 0xffffffff, 0x000b000a,
 605	0x2482, 0xffffffff, 0x000c000f,
 606	0x2483, 0xffffffff, 0x000e000d,
 607	0x2484, 0xffffffff, 0x00110010,
 608	0x2485, 0xffffffff, 0x000a0009,
 609	0x2486, 0xffffffff, 0x000c000b,
 610	0x2487, 0xffffffff, 0x0000000f,
 611	0x2488, 0xffffffff, 0x000e000d,
 612	0x2489, 0xffffffff, 0x00110010,
 613	0x248a, 0xffffffff, 0x000b000a,
 614	0x248b, 0xffffffff, 0x000d000c,
 615	0x248c, 0xffffffff, 0x00000010,
 616	0x248d, 0xffffffff, 0x000f000e,
 617	0x248e, 0xffffffff, 0x00120011,
 618	0x248f, 0xffffffff, 0x000c000b,
 619	0x2490, 0xffffffff, 0x000e000d,
 620	0x2491, 0xffffffff, 0x00000011,
 621	0x2492, 0xffffffff, 0x0010000f,
 622	0x2493, 0xffffffff, 0x00130012,
 623	0x2494, 0xffffffff, 0x000d000c,
 624	0x2495, 0xffffffff, 0x000f000e,
 625	0x2496, 0xffffffff, 0x00100013,
 626	0x2497, 0xffffffff, 0x00120011,
 627	0x2498, 0xffffffff, 0x00150014,
 628	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 629	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 630	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 631	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 632	0x000c, 0xffffffff, 0x0000001c,
 633	0x000d, 0x000f0000, 0x000f0000,
 634	0x0583, 0xffffffff, 0x00000100,
 635	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 636	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 637	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 638	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 639	0x157a, 0x00000001, 0x00000001,
 640	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 641	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 642	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 643	0x3430, 0xfffffff0, 0x00000100,
 644	0x3630, 0xfffffff0, 0x00000100,
 645};
 646
 647static const u32 verde_mgcg_cgcg_init[] =
 648{
 649	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 650	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 651	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 652	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 653	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 654	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 655	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 656	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 657	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 658	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 659	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 660	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 661	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 662	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 663	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 664	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 665	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 666	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 667	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 668	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 669	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 670	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 671	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 672	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 673	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 674	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 675	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 676	0x2458, 0xffffffff, 0x00010000,
 677	0x2459, 0xffffffff, 0x00030002,
 678	0x245a, 0xffffffff, 0x00040007,
 679	0x245b, 0xffffffff, 0x00060005,
 680	0x245c, 0xffffffff, 0x00090008,
 681	0x245d, 0xffffffff, 0x00020001,
 682	0x245e, 0xffffffff, 0x00040003,
 683	0x245f, 0xffffffff, 0x00000007,
 684	0x2460, 0xffffffff, 0x00060005,
 685	0x2461, 0xffffffff, 0x00090008,
 686	0x2462, 0xffffffff, 0x00030002,
 687	0x2463, 0xffffffff, 0x00050004,
 688	0x2464, 0xffffffff, 0x00000008,
 689	0x2465, 0xffffffff, 0x00070006,
 690	0x2466, 0xffffffff, 0x000a0009,
 691	0x2467, 0xffffffff, 0x00040003,
 692	0x2468, 0xffffffff, 0x00060005,
 693	0x2469, 0xffffffff, 0x00000009,
 694	0x246a, 0xffffffff, 0x00080007,
 695	0x246b, 0xffffffff, 0x000b000a,
 696	0x246c, 0xffffffff, 0x00050004,
 697	0x246d, 0xffffffff, 0x00070006,
 698	0x246e, 0xffffffff, 0x0008000b,
 699	0x246f, 0xffffffff, 0x000a0009,
 700	0x2470, 0xffffffff, 0x000d000c,
 701	0x2480, 0xffffffff, 0x00090008,
 702	0x2481, 0xffffffff, 0x000b000a,
 703	0x2482, 0xffffffff, 0x000c000f,
 704	0x2483, 0xffffffff, 0x000e000d,
 705	0x2484, 0xffffffff, 0x00110010,
 706	0x2485, 0xffffffff, 0x000a0009,
 707	0x2486, 0xffffffff, 0x000c000b,
 708	0x2487, 0xffffffff, 0x0000000f,
 709	0x2488, 0xffffffff, 0x000e000d,
 710	0x2489, 0xffffffff, 0x00110010,
 711	0x248a, 0xffffffff, 0x000b000a,
 712	0x248b, 0xffffffff, 0x000d000c,
 713	0x248c, 0xffffffff, 0x00000010,
 714	0x248d, 0xffffffff, 0x000f000e,
 715	0x248e, 0xffffffff, 0x00120011,
 716	0x248f, 0xffffffff, 0x000c000b,
 717	0x2490, 0xffffffff, 0x000e000d,
 718	0x2491, 0xffffffff, 0x00000011,
 719	0x2492, 0xffffffff, 0x0010000f,
 720	0x2493, 0xffffffff, 0x00130012,
 721	0x2494, 0xffffffff, 0x000d000c,
 722	0x2495, 0xffffffff, 0x000f000e,
 723	0x2496, 0xffffffff, 0x00100013,
 724	0x2497, 0xffffffff, 0x00120011,
 725	0x2498, 0xffffffff, 0x00150014,
 726	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 727	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 728	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 729	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 730	0x000c, 0xffffffff, 0x0000001c,
 731	0x000d, 0x000f0000, 0x000f0000,
 732	0x0583, 0xffffffff, 0x00000100,
 733	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 734	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 735	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 736	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 737	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 738	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 739	0x157a, 0x00000001, 0x00000001,
 740	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 741	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 742	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 743	0x3430, 0xfffffff0, 0x00000100,
 744	0x3630, 0xfffffff0, 0x00000100,
 745};
 746
 747static const u32 oland_mgcg_cgcg_init[] =
 748{
 749	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 750	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 751	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 752	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 753	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 754	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 755	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 756	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 757	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 758	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 759	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 760	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 761	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 762	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 763	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 764	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 765	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 766	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 767	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 768	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 769	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 770	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 771	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 772	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 773	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 774	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 775	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 776	0x2458, 0xffffffff, 0x00010000,
 777	0x2459, 0xffffffff, 0x00030002,
 778	0x245a, 0xffffffff, 0x00040007,
 779	0x245b, 0xffffffff, 0x00060005,
 780	0x245c, 0xffffffff, 0x00090008,
 781	0x245d, 0xffffffff, 0x00020001,
 782	0x245e, 0xffffffff, 0x00040003,
 783	0x245f, 0xffffffff, 0x00000007,
 784	0x2460, 0xffffffff, 0x00060005,
 785	0x2461, 0xffffffff, 0x00090008,
 786	0x2462, 0xffffffff, 0x00030002,
 787	0x2463, 0xffffffff, 0x00050004,
 788	0x2464, 0xffffffff, 0x00000008,
 789	0x2465, 0xffffffff, 0x00070006,
 790	0x2466, 0xffffffff, 0x000a0009,
 791	0x2467, 0xffffffff, 0x00040003,
 792	0x2468, 0xffffffff, 0x00060005,
 793	0x2469, 0xffffffff, 0x00000009,
 794	0x246a, 0xffffffff, 0x00080007,
 795	0x246b, 0xffffffff, 0x000b000a,
 796	0x246c, 0xffffffff, 0x00050004,
 797	0x246d, 0xffffffff, 0x00070006,
 798	0x246e, 0xffffffff, 0x0008000b,
 799	0x246f, 0xffffffff, 0x000a0009,
 800	0x2470, 0xffffffff, 0x000d000c,
 801	0x2471, 0xffffffff, 0x00060005,
 802	0x2472, 0xffffffff, 0x00080007,
 803	0x2473, 0xffffffff, 0x0000000b,
 804	0x2474, 0xffffffff, 0x000a0009,
 805	0x2475, 0xffffffff, 0x000d000c,
 806	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 807	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 808	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 809	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 810	0x000c, 0xffffffff, 0x0000001c,
 811	0x000d, 0x000f0000, 0x000f0000,
 812	0x0583, 0xffffffff, 0x00000100,
 813	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 814	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 815	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 816	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 817	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 818	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 819	0x157a, 0x00000001, 0x00000001,
 820	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 821	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 822	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 823	0x3430, 0xfffffff0, 0x00000100,
 824	0x3630, 0xfffffff0, 0x00000100,
 825};
 826
 827static const u32 hainan_mgcg_cgcg_init[] =
 828{
 829	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 830	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 831	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 832	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 833	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 834	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 835	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 836	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 837	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 838	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 839	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 840	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 841	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 842	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 843	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 844	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 845	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 846	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 847	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 848	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 849	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 850	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 851	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 852	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 853	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 854	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 855	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 856	0x2458, 0xffffffff, 0x00010000,
 857	0x2459, 0xffffffff, 0x00030002,
 858	0x245a, 0xffffffff, 0x00040007,
 859	0x245b, 0xffffffff, 0x00060005,
 860	0x245c, 0xffffffff, 0x00090008,
 861	0x245d, 0xffffffff, 0x00020001,
 862	0x245e, 0xffffffff, 0x00040003,
 863	0x245f, 0xffffffff, 0x00000007,
 864	0x2460, 0xffffffff, 0x00060005,
 865	0x2461, 0xffffffff, 0x00090008,
 866	0x2462, 0xffffffff, 0x00030002,
 867	0x2463, 0xffffffff, 0x00050004,
 868	0x2464, 0xffffffff, 0x00000008,
 869	0x2465, 0xffffffff, 0x00070006,
 870	0x2466, 0xffffffff, 0x000a0009,
 871	0x2467, 0xffffffff, 0x00040003,
 872	0x2468, 0xffffffff, 0x00060005,
 873	0x2469, 0xffffffff, 0x00000009,
 874	0x246a, 0xffffffff, 0x00080007,
 875	0x246b, 0xffffffff, 0x000b000a,
 876	0x246c, 0xffffffff, 0x00050004,
 877	0x246d, 0xffffffff, 0x00070006,
 878	0x246e, 0xffffffff, 0x0008000b,
 879	0x246f, 0xffffffff, 0x000a0009,
 880	0x2470, 0xffffffff, 0x000d000c,
 881	0x2471, 0xffffffff, 0x00060005,
 882	0x2472, 0xffffffff, 0x00080007,
 883	0x2473, 0xffffffff, 0x0000000b,
 884	0x2474, 0xffffffff, 0x000a0009,
 885	0x2475, 0xffffffff, 0x000d000c,
 886	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 887	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 888	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 889	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 890	0x000c, 0xffffffff, 0x0000001c,
 891	0x000d, 0x000f0000, 0x000f0000,
 892	0x0583, 0xffffffff, 0x00000100,
 893	0x0409, 0xffffffff, 0x00000100,
 894	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 895	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 896	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 897	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 898	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 899	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 900	0x3430, 0xfffffff0, 0x00000100,
 901	0x3630, 0xfffffff0, 0x00000100,
 902};
 903
 904static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 905{
 906	unsigned long flags;
 907	u32 r;
 908
 909	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 910	WREG32(AMDGPU_PCIE_INDEX, reg);
 911	(void)RREG32(AMDGPU_PCIE_INDEX);
 912	r = RREG32(AMDGPU_PCIE_DATA);
 913	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 914	return r;
 915}
 916
 917static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 918{
 919	unsigned long flags;
 920
 921	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 922	WREG32(AMDGPU_PCIE_INDEX, reg);
 923	(void)RREG32(AMDGPU_PCIE_INDEX);
 924	WREG32(AMDGPU_PCIE_DATA, v);
 925	(void)RREG32(AMDGPU_PCIE_DATA);
 926	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 927}
 928
 929static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 930{
 931	unsigned long flags;
 932	u32 r;
 933
 934	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 935	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 936	(void)RREG32(PCIE_PORT_INDEX);
 937	r = RREG32(PCIE_PORT_DATA);
 938	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 939	return r;
 940}
 941
 942static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 943{
 944	unsigned long flags;
 945
 946	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 947	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 948	(void)RREG32(PCIE_PORT_INDEX);
 949	WREG32(PCIE_PORT_DATA, (v));
 950	(void)RREG32(PCIE_PORT_DATA);
 951	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 952}
 953
 954static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 955{
 956	unsigned long flags;
 957	u32 r;
 958
 959	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 960	WREG32(SMC_IND_INDEX_0, (reg));
 961	r = RREG32(SMC_IND_DATA_0);
 962	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 963	return r;
 964}
 965
 966static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 967{
 968	unsigned long flags;
 969
 970	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 971	WREG32(SMC_IND_INDEX_0, (reg));
 972	WREG32(SMC_IND_DATA_0, (v));
 973	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 974}
 975
 976static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
 977	{GRBM_STATUS},
 978	{GB_ADDR_CONFIG},
 979	{MC_ARB_RAMCFG},
 980	{GB_TILE_MODE0},
 981	{GB_TILE_MODE1},
 982	{GB_TILE_MODE2},
 983	{GB_TILE_MODE3},
 984	{GB_TILE_MODE4},
 985	{GB_TILE_MODE5},
 986	{GB_TILE_MODE6},
 987	{GB_TILE_MODE7},
 988	{GB_TILE_MODE8},
 989	{GB_TILE_MODE9},
 990	{GB_TILE_MODE10},
 991	{GB_TILE_MODE11},
 992	{GB_TILE_MODE12},
 993	{GB_TILE_MODE13},
 994	{GB_TILE_MODE14},
 995	{GB_TILE_MODE15},
 996	{GB_TILE_MODE16},
 997	{GB_TILE_MODE17},
 998	{GB_TILE_MODE18},
 999	{GB_TILE_MODE19},
1000	{GB_TILE_MODE20},
1001	{GB_TILE_MODE21},
1002	{GB_TILE_MODE22},
1003	{GB_TILE_MODE23},
1004	{GB_TILE_MODE24},
1005	{GB_TILE_MODE25},
1006	{GB_TILE_MODE26},
1007	{GB_TILE_MODE27},
1008	{GB_TILE_MODE28},
1009	{GB_TILE_MODE29},
1010	{GB_TILE_MODE30},
1011	{GB_TILE_MODE31},
1012	{CC_RB_BACKEND_DISABLE, true},
1013	{GC_USER_RB_BACKEND_DISABLE, true},
1014	{PA_SC_RASTER_CONFIG, true},
1015};
1016
1017static uint32_t si_get_register_value(struct amdgpu_device *adev,
1018				      bool indexed, u32 se_num,
1019				      u32 sh_num, u32 reg_offset)
1020{
1021	if (indexed) {
1022		uint32_t val;
1023		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1024		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1025
1026		switch (reg_offset) {
1027		case mmCC_RB_BACKEND_DISABLE:
1028			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1029		case mmGC_USER_RB_BACKEND_DISABLE:
1030			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1031		case mmPA_SC_RASTER_CONFIG:
1032			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1033		}
1034
1035		mutex_lock(&adev->grbm_idx_mutex);
1036		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1037			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1038
1039		val = RREG32(reg_offset);
1040
1041		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1042			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1043		mutex_unlock(&adev->grbm_idx_mutex);
1044		return val;
1045	} else {
1046		unsigned idx;
1047
1048		switch (reg_offset) {
1049		case mmGB_ADDR_CONFIG:
1050			return adev->gfx.config.gb_addr_config;
1051		case mmMC_ARB_RAMCFG:
1052			return adev->gfx.config.mc_arb_ramcfg;
1053		case mmGB_TILE_MODE0:
1054		case mmGB_TILE_MODE1:
1055		case mmGB_TILE_MODE2:
1056		case mmGB_TILE_MODE3:
1057		case mmGB_TILE_MODE4:
1058		case mmGB_TILE_MODE5:
1059		case mmGB_TILE_MODE6:
1060		case mmGB_TILE_MODE7:
1061		case mmGB_TILE_MODE8:
1062		case mmGB_TILE_MODE9:
1063		case mmGB_TILE_MODE10:
1064		case mmGB_TILE_MODE11:
1065		case mmGB_TILE_MODE12:
1066		case mmGB_TILE_MODE13:
1067		case mmGB_TILE_MODE14:
1068		case mmGB_TILE_MODE15:
1069		case mmGB_TILE_MODE16:
1070		case mmGB_TILE_MODE17:
1071		case mmGB_TILE_MODE18:
1072		case mmGB_TILE_MODE19:
1073		case mmGB_TILE_MODE20:
1074		case mmGB_TILE_MODE21:
1075		case mmGB_TILE_MODE22:
1076		case mmGB_TILE_MODE23:
1077		case mmGB_TILE_MODE24:
1078		case mmGB_TILE_MODE25:
1079		case mmGB_TILE_MODE26:
1080		case mmGB_TILE_MODE27:
1081		case mmGB_TILE_MODE28:
1082		case mmGB_TILE_MODE29:
1083		case mmGB_TILE_MODE30:
1084		case mmGB_TILE_MODE31:
1085			idx = (reg_offset - mmGB_TILE_MODE0);
1086			return adev->gfx.config.tile_mode_array[idx];
1087		default:
1088			return RREG32(reg_offset);
1089		}
1090	}
1091}
1092static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1093			     u32 sh_num, u32 reg_offset, u32 *value)
1094{
1095	uint32_t i;
1096
1097	*value = 0;
1098	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1099		bool indexed = si_allowed_read_registers[i].grbm_indexed;
1100
1101		if (reg_offset != si_allowed_read_registers[i].reg_offset)
1102			continue;
1103
1104		*value = si_get_register_value(adev, indexed, se_num, sh_num,
1105					       reg_offset);
1106		return 0;
1107	}
1108	return -EINVAL;
1109}
1110
1111static bool si_read_disabled_bios(struct amdgpu_device *adev)
1112{
1113	u32 bus_cntl;
1114	u32 d1vga_control = 0;
1115	u32 d2vga_control = 0;
1116	u32 vga_render_control = 0;
1117	u32 rom_cntl;
1118	bool r;
1119
1120	bus_cntl = RREG32(R600_BUS_CNTL);
1121	if (adev->mode_info.num_crtc) {
1122		d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1123		d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1124		vga_render_control = RREG32(VGA_RENDER_CONTROL);
1125	}
1126	rom_cntl = RREG32(R600_ROM_CNTL);
1127
1128	/* enable the rom */
1129	WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1130	if (adev->mode_info.num_crtc) {
1131		/* Disable VGA mode */
1132		WREG32(AVIVO_D1VGA_CONTROL,
1133		       (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1134					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1135		WREG32(AVIVO_D2VGA_CONTROL,
1136		       (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1137					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1138		WREG32(VGA_RENDER_CONTROL,
1139		       (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1140	}
1141	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1142
1143	r = amdgpu_read_bios(adev);
1144
1145	/* restore regs */
1146	WREG32(R600_BUS_CNTL, bus_cntl);
1147	if (adev->mode_info.num_crtc) {
1148		WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1149		WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1150		WREG32(VGA_RENDER_CONTROL, vga_render_control);
1151	}
1152	WREG32(R600_ROM_CNTL, rom_cntl);
1153	return r;
1154}
1155
1156#define mmROM_INDEX 0x2A
1157#define mmROM_DATA  0x2B
1158
1159static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1160				  u8 *bios, u32 length_bytes)
1161{
1162	u32 *dw_ptr;
1163	u32 i, length_dw;
1164
1165	if (bios == NULL)
1166		return false;
1167	if (length_bytes == 0)
1168		return false;
1169	/* APU vbios image is part of sbios image */
1170	if (adev->flags & AMD_IS_APU)
1171		return false;
1172
1173	dw_ptr = (u32 *)bios;
1174	length_dw = ALIGN(length_bytes, 4) / 4;
1175	/* set rom index to 0 */
1176	WREG32(mmROM_INDEX, 0);
1177	for (i = 0; i < length_dw; i++)
1178		dw_ptr[i] = RREG32(mmROM_DATA);
1179
1180	return true;
1181}
1182
1183//xxx: not implemented
1184static int si_asic_reset(struct amdgpu_device *adev)
1185{
1186	return 0;
1187}
1188
1189static enum amd_reset_method
1190si_asic_reset_method(struct amdgpu_device *adev)
1191{
1192	return AMD_RESET_METHOD_LEGACY;
1193}
1194
1195static u32 si_get_config_memsize(struct amdgpu_device *adev)
1196{
1197	return RREG32(mmCONFIG_MEMSIZE);
1198}
1199
1200static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1201{
1202	uint32_t temp;
1203
1204	temp = RREG32(CONFIG_CNTL);
1205	if (state == false) {
1206		temp &= ~(1<<0);
1207		temp |= (1<<1);
1208	} else {
1209		temp &= ~(1<<1);
1210	}
1211	WREG32(CONFIG_CNTL, temp);
1212}
1213
1214static u32 si_get_xclk(struct amdgpu_device *adev)
1215{
1216        u32 reference_clock = adev->clock.spll.reference_freq;
1217	u32 tmp;
1218
1219	tmp = RREG32(CG_CLKPIN_CNTL_2);
1220	if (tmp & MUX_TCLK_TO_XCLK)
1221		return TCLK;
1222
1223	tmp = RREG32(CG_CLKPIN_CNTL);
1224	if (tmp & XTALIN_DIVIDE)
1225		return reference_clock / 4;
1226
1227	return reference_clock;
1228}
1229
1230//xxx:not implemented
1231static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1232{
1233	return 0;
1234}
1235
1236static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1237{
1238	if (is_virtual_machine()) /* passthrough mode */
1239		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1240}
1241
1242static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1243{
1244	if (!ring || !ring->funcs->emit_wreg) {
1245		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1246		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1247	} else {
1248		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1249	}
1250}
1251
1252static void si_invalidate_hdp(struct amdgpu_device *adev,
1253			      struct amdgpu_ring *ring)
1254{
1255	if (!ring || !ring->funcs->emit_wreg) {
1256		WREG32(mmHDP_DEBUG0, 1);
1257		RREG32(mmHDP_DEBUG0);
1258	} else {
1259		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1260	}
1261}
1262
1263static bool si_need_full_reset(struct amdgpu_device *adev)
1264{
1265	/* change this when we support soft reset */
1266	return true;
1267}
1268
1269static bool si_need_reset_on_init(struct amdgpu_device *adev)
1270{
1271	return false;
1272}
1273
1274static int si_get_pcie_lanes(struct amdgpu_device *adev)
1275{
1276	u32 link_width_cntl;
1277
1278	if (adev->flags & AMD_IS_APU)
1279		return 0;
1280
1281	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1282
1283	switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {
1284	case LC_LINK_WIDTH_X1:
1285		return 1;
1286	case LC_LINK_WIDTH_X2:
1287		return 2;
1288	case LC_LINK_WIDTH_X4:
1289		return 4;
1290	case LC_LINK_WIDTH_X8:
1291		return 8;
1292	case LC_LINK_WIDTH_X0:
1293	case LC_LINK_WIDTH_X16:
1294	default:
1295		return 16;
1296	}
1297}
1298
1299static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1300{
1301	u32 link_width_cntl, mask;
1302
1303	if (adev->flags & AMD_IS_APU)
1304		return;
1305
1306	switch (lanes) {
1307	case 0:
1308		mask = LC_LINK_WIDTH_X0;
1309		break;
1310	case 1:
1311		mask = LC_LINK_WIDTH_X1;
1312		break;
1313	case 2:
1314		mask = LC_LINK_WIDTH_X2;
1315		break;
1316	case 4:
1317		mask = LC_LINK_WIDTH_X4;
1318		break;
1319	case 8:
1320		mask = LC_LINK_WIDTH_X8;
1321		break;
1322	case 16:
1323		mask = LC_LINK_WIDTH_X16;
1324		break;
1325	default:
1326		DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1327		return;
1328	}
1329
1330	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1331	link_width_cntl &= ~LC_LINK_WIDTH_MASK;
1332	link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;
1333	link_width_cntl |= (LC_RECONFIG_NOW |
1334			    LC_RECONFIG_ARC_MISSING_ESCAPE);
1335
1336	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1337}
1338
1339static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1340			      uint64_t *count1)
1341{
1342	uint32_t perfctr = 0;
1343	uint64_t cnt0_of, cnt1_of;
1344	int tmp;
1345
1346	/* This reports 0 on APUs, so return to avoid writing/reading registers
1347	 * that may or may not be different from their GPU counterparts
1348	 */
1349	if (adev->flags & AMD_IS_APU)
1350		return;
1351
1352	/* Set the 2 events that we wish to watch, defined above */
1353	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1354	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1355	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1356
1357	/* Write to enable desired perf counters */
1358	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1359	/* Zero out and enable the perf counters
1360	 * Write 0x5:
1361	 * Bit 0 = Start all counters(1)
1362	 * Bit 2 = Global counter reset enable(1)
1363	 */
1364	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1365
1366	msleep(1000);
1367
1368	/* Load the shadow and disable the perf counters
1369	 * Write 0x2:
1370	 * Bit 0 = Stop counters(0)
1371	 * Bit 1 = Load the shadow counters(1)
1372	 */
1373	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1374
1375	/* Read register values to get any >32bit overflow */
1376	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1377	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1378	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1379
1380	/* Get the values and add the overflow */
1381	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1382	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1383}
1384
1385static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1386{
1387	uint64_t nak_r, nak_g;
1388
1389	/* Get the number of NAKs received and generated */
1390	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1391	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1392
1393	/* Add the total number of NAKs, i.e the number of replays */
1394	return (nak_r + nak_g);
1395}
1396
1397static const struct amdgpu_asic_funcs si_asic_funcs =
1398{
1399	.read_disabled_bios = &si_read_disabled_bios,
1400	.read_bios_from_rom = &si_read_bios_from_rom,
1401	.read_register = &si_read_register,
1402	.reset = &si_asic_reset,
1403	.reset_method = &si_asic_reset_method,
1404	.set_vga_state = &si_vga_set_state,
1405	.get_xclk = &si_get_xclk,
1406	.set_uvd_clocks = &si_set_uvd_clocks,
1407	.set_vce_clocks = NULL,
1408	.get_pcie_lanes = &si_get_pcie_lanes,
1409	.set_pcie_lanes = &si_set_pcie_lanes,
1410	.get_config_memsize = &si_get_config_memsize,
1411	.flush_hdp = &si_flush_hdp,
1412	.invalidate_hdp = &si_invalidate_hdp,
1413	.need_full_reset = &si_need_full_reset,
1414	.get_pcie_usage = &si_get_pcie_usage,
1415	.need_reset_on_init = &si_need_reset_on_init,
1416	.get_pcie_replay_count = &si_get_pcie_replay_count,
1417};
1418
1419static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1420{
1421	return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1422		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1423}
1424
1425static int si_common_early_init(void *handle)
1426{
1427	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1428
1429	adev->smc_rreg = &si_smc_rreg;
1430	adev->smc_wreg = &si_smc_wreg;
1431	adev->pcie_rreg = &si_pcie_rreg;
1432	adev->pcie_wreg = &si_pcie_wreg;
1433	adev->pciep_rreg = &si_pciep_rreg;
1434	adev->pciep_wreg = &si_pciep_wreg;
1435	adev->uvd_ctx_rreg = NULL;
1436	adev->uvd_ctx_wreg = NULL;
1437	adev->didt_rreg = NULL;
1438	adev->didt_wreg = NULL;
1439
1440	adev->asic_funcs = &si_asic_funcs;
1441
1442	adev->rev_id = si_get_rev_id(adev);
1443	adev->external_rev_id = 0xFF;
1444	switch (adev->asic_type) {
1445	case CHIP_TAHITI:
1446		adev->cg_flags =
1447			AMD_CG_SUPPORT_GFX_MGCG |
1448			AMD_CG_SUPPORT_GFX_MGLS |
1449			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1450			AMD_CG_SUPPORT_GFX_CGLS |
1451			AMD_CG_SUPPORT_GFX_CGTS |
1452			AMD_CG_SUPPORT_GFX_CP_LS |
1453			AMD_CG_SUPPORT_MC_MGCG |
1454			AMD_CG_SUPPORT_SDMA_MGCG |
1455			AMD_CG_SUPPORT_BIF_LS |
1456			AMD_CG_SUPPORT_VCE_MGCG |
1457			AMD_CG_SUPPORT_UVD_MGCG |
1458			AMD_CG_SUPPORT_HDP_LS |
1459			AMD_CG_SUPPORT_HDP_MGCG;
1460		adev->pg_flags = 0;
1461		adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1462					(adev->rev_id == 1) ? 5 : 6;
1463		break;
1464	case CHIP_PITCAIRN:
1465		adev->cg_flags =
1466			AMD_CG_SUPPORT_GFX_MGCG |
1467			AMD_CG_SUPPORT_GFX_MGLS |
1468			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1469			AMD_CG_SUPPORT_GFX_CGLS |
1470			AMD_CG_SUPPORT_GFX_CGTS |
1471			AMD_CG_SUPPORT_GFX_CP_LS |
1472			AMD_CG_SUPPORT_GFX_RLC_LS |
1473			AMD_CG_SUPPORT_MC_LS |
1474			AMD_CG_SUPPORT_MC_MGCG |
1475			AMD_CG_SUPPORT_SDMA_MGCG |
1476			AMD_CG_SUPPORT_BIF_LS |
1477			AMD_CG_SUPPORT_VCE_MGCG |
1478			AMD_CG_SUPPORT_UVD_MGCG |
1479			AMD_CG_SUPPORT_HDP_LS |
1480			AMD_CG_SUPPORT_HDP_MGCG;
1481		adev->pg_flags = 0;
1482		adev->external_rev_id = adev->rev_id + 20;
1483		break;
1484
1485	case CHIP_VERDE:
1486		adev->cg_flags =
1487			AMD_CG_SUPPORT_GFX_MGCG |
1488			AMD_CG_SUPPORT_GFX_MGLS |
1489			AMD_CG_SUPPORT_GFX_CGLS |
1490			AMD_CG_SUPPORT_GFX_CGTS |
1491			AMD_CG_SUPPORT_GFX_CGTS_LS |
1492			AMD_CG_SUPPORT_GFX_CP_LS |
1493			AMD_CG_SUPPORT_MC_LS |
1494			AMD_CG_SUPPORT_MC_MGCG |
1495			AMD_CG_SUPPORT_SDMA_MGCG |
1496			AMD_CG_SUPPORT_SDMA_LS |
1497			AMD_CG_SUPPORT_BIF_LS |
1498			AMD_CG_SUPPORT_VCE_MGCG |
1499			AMD_CG_SUPPORT_UVD_MGCG |
1500			AMD_CG_SUPPORT_HDP_LS |
1501			AMD_CG_SUPPORT_HDP_MGCG;
1502		adev->pg_flags = 0;
1503		//???
1504		adev->external_rev_id = adev->rev_id + 40;
1505		break;
1506	case CHIP_OLAND:
1507		adev->cg_flags =
1508			AMD_CG_SUPPORT_GFX_MGCG |
1509			AMD_CG_SUPPORT_GFX_MGLS |
1510			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1511			AMD_CG_SUPPORT_GFX_CGLS |
1512			AMD_CG_SUPPORT_GFX_CGTS |
1513			AMD_CG_SUPPORT_GFX_CP_LS |
1514			AMD_CG_SUPPORT_GFX_RLC_LS |
1515			AMD_CG_SUPPORT_MC_LS |
1516			AMD_CG_SUPPORT_MC_MGCG |
1517			AMD_CG_SUPPORT_SDMA_MGCG |
1518			AMD_CG_SUPPORT_BIF_LS |
1519			AMD_CG_SUPPORT_UVD_MGCG |
1520			AMD_CG_SUPPORT_HDP_LS |
1521			AMD_CG_SUPPORT_HDP_MGCG;
1522		adev->pg_flags = 0;
1523		adev->external_rev_id = 60;
1524		break;
1525	case CHIP_HAINAN:
1526		adev->cg_flags =
1527			AMD_CG_SUPPORT_GFX_MGCG |
1528			AMD_CG_SUPPORT_GFX_MGLS |
1529			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1530			AMD_CG_SUPPORT_GFX_CGLS |
1531			AMD_CG_SUPPORT_GFX_CGTS |
1532			AMD_CG_SUPPORT_GFX_CP_LS |
1533			AMD_CG_SUPPORT_GFX_RLC_LS |
1534			AMD_CG_SUPPORT_MC_LS |
1535			AMD_CG_SUPPORT_MC_MGCG |
1536			AMD_CG_SUPPORT_SDMA_MGCG |
1537			AMD_CG_SUPPORT_BIF_LS |
1538			AMD_CG_SUPPORT_HDP_LS |
1539			AMD_CG_SUPPORT_HDP_MGCG;
1540		adev->pg_flags = 0;
1541		adev->external_rev_id = 70;
1542		break;
1543
1544	default:
1545		return -EINVAL;
1546	}
1547
1548	return 0;
1549}
1550
1551static int si_common_sw_init(void *handle)
1552{
1553	return 0;
1554}
1555
1556static int si_common_sw_fini(void *handle)
1557{
1558	return 0;
1559}
1560
1561
1562static void si_init_golden_registers(struct amdgpu_device *adev)
1563{
1564	switch (adev->asic_type) {
1565	case CHIP_TAHITI:
1566		amdgpu_device_program_register_sequence(adev,
1567							tahiti_golden_registers,
1568							ARRAY_SIZE(tahiti_golden_registers));
1569		amdgpu_device_program_register_sequence(adev,
1570							tahiti_golden_rlc_registers,
1571							ARRAY_SIZE(tahiti_golden_rlc_registers));
1572		amdgpu_device_program_register_sequence(adev,
1573							tahiti_mgcg_cgcg_init,
1574							ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1575		amdgpu_device_program_register_sequence(adev,
1576							tahiti_golden_registers2,
1577							ARRAY_SIZE(tahiti_golden_registers2));
1578		break;
1579	case CHIP_PITCAIRN:
1580		amdgpu_device_program_register_sequence(adev,
1581							pitcairn_golden_registers,
1582							ARRAY_SIZE(pitcairn_golden_registers));
1583		amdgpu_device_program_register_sequence(adev,
1584							pitcairn_golden_rlc_registers,
1585							ARRAY_SIZE(pitcairn_golden_rlc_registers));
1586		amdgpu_device_program_register_sequence(adev,
1587							pitcairn_mgcg_cgcg_init,
1588							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1589		break;
1590	case CHIP_VERDE:
1591		amdgpu_device_program_register_sequence(adev,
1592							verde_golden_registers,
1593							ARRAY_SIZE(verde_golden_registers));
1594		amdgpu_device_program_register_sequence(adev,
1595							verde_golden_rlc_registers,
1596							ARRAY_SIZE(verde_golden_rlc_registers));
1597		amdgpu_device_program_register_sequence(adev,
1598							verde_mgcg_cgcg_init,
1599							ARRAY_SIZE(verde_mgcg_cgcg_init));
1600		amdgpu_device_program_register_sequence(adev,
1601							verde_pg_init,
1602							ARRAY_SIZE(verde_pg_init));
1603		break;
1604	case CHIP_OLAND:
1605		amdgpu_device_program_register_sequence(adev,
1606							oland_golden_registers,
1607							ARRAY_SIZE(oland_golden_registers));
1608		amdgpu_device_program_register_sequence(adev,
1609							oland_golden_rlc_registers,
1610							ARRAY_SIZE(oland_golden_rlc_registers));
1611		amdgpu_device_program_register_sequence(adev,
1612							oland_mgcg_cgcg_init,
1613							ARRAY_SIZE(oland_mgcg_cgcg_init));
1614		break;
1615	case CHIP_HAINAN:
1616		amdgpu_device_program_register_sequence(adev,
1617							hainan_golden_registers,
1618							ARRAY_SIZE(hainan_golden_registers));
1619		amdgpu_device_program_register_sequence(adev,
1620							hainan_golden_registers2,
1621							ARRAY_SIZE(hainan_golden_registers2));
1622		amdgpu_device_program_register_sequence(adev,
1623							hainan_mgcg_cgcg_init,
1624							ARRAY_SIZE(hainan_mgcg_cgcg_init));
1625		break;
1626
1627
1628	default:
1629		BUG();
1630	}
1631}
1632
1633static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1634{
1635	struct pci_dev *root = adev->pdev->bus->self;
1636	int bridge_pos, gpu_pos;
1637	u32 speed_cntl, current_data_rate;
1638	int i;
1639	u16 tmp16;
1640
1641	if (pci_is_root_bus(adev->pdev->bus))
1642		return;
1643
1644	if (amdgpu_pcie_gen2 == 0)
1645		return;
1646
1647	if (adev->flags & AMD_IS_APU)
1648		return;
1649
1650	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1651					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1652		return;
1653
1654	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1655	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1656		LC_CURRENT_DATA_RATE_SHIFT;
1657	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1658		if (current_data_rate == 2) {
1659			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1660			return;
1661		}
1662		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1663	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1664		if (current_data_rate == 1) {
1665			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1666			return;
1667		}
1668		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1669	}
1670
1671	bridge_pos = pci_pcie_cap(root);
1672	if (!bridge_pos)
1673		return;
1674
1675	gpu_pos = pci_pcie_cap(adev->pdev);
1676	if (!gpu_pos)
1677		return;
1678
1679	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1680		if (current_data_rate != 2) {
1681			u16 bridge_cfg, gpu_cfg;
1682			u16 bridge_cfg2, gpu_cfg2;
1683			u32 max_lw, current_lw, tmp;
1684
1685			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1686			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1687
1688			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1689			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1690
1691			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1692			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1693
1694			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1695			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1696			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1697
1698			if (current_lw < max_lw) {
1699				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1700				if (tmp & LC_RENEGOTIATION_SUPPORT) {
1701					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1702					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1703					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1704					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1705				}
1706			}
1707
1708			for (i = 0; i < 10; i++) {
1709				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1710				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1711					break;
1712
1713				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1714				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1715
1716				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1717				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1718
1719				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1720				tmp |= LC_SET_QUIESCE;
1721				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1722
1723				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1724				tmp |= LC_REDO_EQ;
1725				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1726
1727				mdelay(100);
1728
1729				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1730				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1731				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1732				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1733
1734				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1735				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1736				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1737				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1738
1739				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1740				tmp16 &= ~((1 << 4) | (7 << 9));
1741				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1742				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1743
1744				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1745				tmp16 &= ~((1 << 4) | (7 << 9));
1746				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1747				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1748
1749				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1750				tmp &= ~LC_SET_QUIESCE;
1751				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1752			}
1753		}
1754	}
1755
1756	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1757	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1758	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1759
1760	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1761	tmp16 &= ~0xf;
1762	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1763		tmp16 |= 3;
1764	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1765		tmp16 |= 2;
1766	else
1767		tmp16 |= 1;
1768	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1769
1770	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1771	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1772	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1773
1774	for (i = 0; i < adev->usec_timeout; i++) {
1775		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1776		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1777			break;
1778		udelay(1);
1779	}
1780}
1781
1782static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1783{
1784	unsigned long flags;
1785	u32 r;
1786
1787	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1788	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1789	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1790	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1791	return r;
1792}
1793
1794static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1795{
1796	unsigned long flags;
1797
1798	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1799	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1800	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1801	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1802}
1803
1804static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1805{
1806	unsigned long flags;
1807	u32 r;
1808
1809	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1810	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1811	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1812	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1813	return r;
1814}
1815
1816static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1817{
1818	unsigned long flags;
1819
1820	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1821	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1822	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1823	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1824}
1825static void si_program_aspm(struct amdgpu_device *adev)
1826{
1827	u32 data, orig;
1828	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1829	bool disable_clkreq = false;
1830
1831	if (amdgpu_aspm == 0)
1832		return;
1833
1834	if (adev->flags & AMD_IS_APU)
1835		return;
1836	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1837	data &= ~LC_XMIT_N_FTS_MASK;
1838	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1839	if (orig != data)
1840		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1841
1842	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1843	data |= LC_GO_TO_RECOVERY;
1844	if (orig != data)
1845		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1846
1847	orig = data = RREG32_PCIE(PCIE_P_CNTL);
1848	data |= P_IGNORE_EDB_ERR;
1849	if (orig != data)
1850		WREG32_PCIE(PCIE_P_CNTL, data);
1851
1852	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1853	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1854	data |= LC_PMI_TO_L1_DIS;
1855	if (!disable_l0s)
1856		data |= LC_L0S_INACTIVITY(7);
1857
1858	if (!disable_l1) {
1859		data |= LC_L1_INACTIVITY(7);
1860		data &= ~LC_PMI_TO_L1_DIS;
1861		if (orig != data)
1862			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1863
1864		if (!disable_plloff_in_l1) {
1865			bool clk_req_support;
1866
1867			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1868			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1869			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1870			if (orig != data)
1871				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1872
1873			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1874			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1875			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1876			if (orig != data)
1877				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1878
1879			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1880			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1881			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1882			if (orig != data)
1883				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1884
1885			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1886			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1887			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1888			if (orig != data)
1889				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1890
1891			if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
1892				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1893				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1894				if (orig != data)
1895					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1896
1897				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1898				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1899				if (orig != data)
1900					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1901
1902				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1903				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1904				if (orig != data)
1905					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1906
1907				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1908				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1909				if (orig != data)
1910					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1911
1912				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1913				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1914				if (orig != data)
1915					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1916
1917				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1918				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1919				if (orig != data)
1920					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1921
1922				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1923				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1924				if (orig != data)
1925					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1926
1927				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1928				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1929				if (orig != data)
1930					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1931			}
1932			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1933			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1934			data |= LC_DYN_LANES_PWR_STATE(3);
1935			if (orig != data)
1936				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1937
1938			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1939			data &= ~LS2_EXIT_TIME_MASK;
1940			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
1941				data |= LS2_EXIT_TIME(5);
1942			if (orig != data)
1943				si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1944
1945			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1946			data &= ~LS2_EXIT_TIME_MASK;
1947			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
1948				data |= LS2_EXIT_TIME(5);
1949			if (orig != data)
1950				si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1951
1952			if (!disable_clkreq &&
1953			    !pci_is_root_bus(adev->pdev->bus)) {
1954				struct pci_dev *root = adev->pdev->bus->self;
1955				u32 lnkcap;
1956
1957				clk_req_support = false;
1958				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1959				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1960					clk_req_support = true;
1961			} else {
1962				clk_req_support = false;
1963			}
1964
1965			if (clk_req_support) {
1966				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
1967				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1968				if (orig != data)
1969					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
1970
1971				orig = data = RREG32(THM_CLK_CNTL);
1972				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1973				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1974				if (orig != data)
1975					WREG32(THM_CLK_CNTL, data);
1976
1977				orig = data = RREG32(MISC_CLK_CNTL);
1978				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1979				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1980				if (orig != data)
1981					WREG32(MISC_CLK_CNTL, data);
1982
1983				orig = data = RREG32(CG_CLKPIN_CNTL);
1984				data &= ~BCLK_AS_XCLK;
1985				if (orig != data)
1986					WREG32(CG_CLKPIN_CNTL, data);
1987
1988				orig = data = RREG32(CG_CLKPIN_CNTL_2);
1989				data &= ~FORCE_BIF_REFCLK_EN;
1990				if (orig != data)
1991					WREG32(CG_CLKPIN_CNTL_2, data);
1992
1993				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1994				data &= ~MPLL_CLKOUT_SEL_MASK;
1995				data |= MPLL_CLKOUT_SEL(4);
1996				if (orig != data)
1997					WREG32(MPLL_BYPASSCLK_SEL, data);
1998
1999				orig = data = RREG32(SPLL_CNTL_MODE);
2000				data &= ~SPLL_REFCLK_SEL_MASK;
2001				if (orig != data)
2002					WREG32(SPLL_CNTL_MODE, data);
2003			}
2004		}
2005	} else {
2006		if (orig != data)
2007			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2008	}
2009
2010	orig = data = RREG32_PCIE(PCIE_CNTL2);
2011	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
2012	if (orig != data)
2013		WREG32_PCIE(PCIE_CNTL2, data);
2014
2015	if (!disable_l0s) {
2016		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
2017		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
2018			data = RREG32_PCIE(PCIE_LC_STATUS1);
2019			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
2020				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
2021				data &= ~LC_L0S_INACTIVITY_MASK;
2022				if (orig != data)
2023					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
2024			}
2025		}
2026	}
2027}
2028
2029static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2030{
2031	int readrq;
2032	u16 v;
2033
2034	readrq = pcie_get_readrq(adev->pdev);
2035	v = ffs(readrq) - 8;
2036	if ((v == 0) || (v == 6) || (v == 7))
2037		pcie_set_readrq(adev->pdev, 512);
2038}
2039
2040static int si_common_hw_init(void *handle)
2041{
2042	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2043
2044	si_fix_pci_max_read_req_size(adev);
2045	si_init_golden_registers(adev);
2046	si_pcie_gen3_enable(adev);
2047	si_program_aspm(adev);
2048
2049	return 0;
2050}
2051
2052static int si_common_hw_fini(void *handle)
2053{
2054	return 0;
2055}
2056
2057static int si_common_suspend(void *handle)
2058{
2059	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2060
2061	return si_common_hw_fini(adev);
2062}
2063
2064static int si_common_resume(void *handle)
2065{
2066	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2067
2068	return si_common_hw_init(adev);
2069}
2070
2071static bool si_common_is_idle(void *handle)
2072{
2073	return true;
2074}
2075
2076static int si_common_wait_for_idle(void *handle)
2077{
2078	return 0;
2079}
2080
2081static int si_common_soft_reset(void *handle)
2082{
2083	return 0;
2084}
2085
2086static int si_common_set_clockgating_state(void *handle,
2087					    enum amd_clockgating_state state)
2088{
2089	return 0;
2090}
2091
2092static int si_common_set_powergating_state(void *handle,
2093					    enum amd_powergating_state state)
2094{
2095	return 0;
2096}
2097
2098static const struct amd_ip_funcs si_common_ip_funcs = {
2099	.name = "si_common",
2100	.early_init = si_common_early_init,
2101	.late_init = NULL,
2102	.sw_init = si_common_sw_init,
2103	.sw_fini = si_common_sw_fini,
2104	.hw_init = si_common_hw_init,
2105	.hw_fini = si_common_hw_fini,
2106	.suspend = si_common_suspend,
2107	.resume = si_common_resume,
2108	.is_idle = si_common_is_idle,
2109	.wait_for_idle = si_common_wait_for_idle,
2110	.soft_reset = si_common_soft_reset,
2111	.set_clockgating_state = si_common_set_clockgating_state,
2112	.set_powergating_state = si_common_set_powergating_state,
2113};
2114
2115static const struct amdgpu_ip_block_version si_common_ip_block =
2116{
2117	.type = AMD_IP_BLOCK_TYPE_COMMON,
2118	.major = 1,
2119	.minor = 0,
2120	.rev = 0,
2121	.funcs = &si_common_ip_funcs,
2122};
2123
2124int si_set_ip_blocks(struct amdgpu_device *adev)
2125{
2126	si_detect_hw_virtualization(adev);
2127
2128	switch (adev->asic_type) {
2129	case CHIP_VERDE:
2130	case CHIP_TAHITI:
2131	case CHIP_PITCAIRN:
2132		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2133		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2134		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2135		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2136		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2137		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2138		if (adev->enable_virtual_display)
2139			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2140		else
2141			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
 
 
2142		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
2143		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2144		break;
2145	case CHIP_OLAND:
2146		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2147		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2148		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2149		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2150		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2151		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2152		if (adev->enable_virtual_display)
2153			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2154		else
2155			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2156
 
2157		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
2158		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2159		break;
2160	case CHIP_HAINAN:
2161		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2162		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2163		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2164		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2165		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2166		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2167		if (adev->enable_virtual_display)
2168			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 
 
2169		break;
2170	default:
2171		BUG();
2172	}
2173	return 0;
2174}
2175