1/*
   2 * Copyright 2015 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <drm/drmP.h>
  28#include "amdgpu.h"
  29#include "amdgpu_atombios.h"
  30#include "amdgpu_ih.h"
  31#include "amdgpu_uvd.h"
  32#include "amdgpu_vce.h"
  33#include "atom.h"
  34#include "amd_pcie.h"
  35#include "si_dpm.h"
  36#include "sid.h"
  37#include "si_ih.h"
  38#include "gfx_v6_0.h"
  39#include "gmc_v6_0.h"
  40#include "si_dma.h"
  41#include "dce_v6_0.h"
  42#include "si.h"
  43#include "dce_virtual.h"
  44#include "gca/gfx_6_0_d.h"
  45#include "oss/oss_1_0_d.h"
  46#include "gmc/gmc_6_0_d.h"
  47#include "dce/dce_6_0_d.h"
  48#include "uvd/uvd_4_0_d.h"
  49#include "bif/bif_3_0_d.h"
  50
  51static const u32 tahiti_golden_registers[] =
  52{
  53	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  54	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  55	mmDB_DEBUG, 0xffffffff, 0x00000000,
  56	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  57	mmDB_DEBUG3, 0x0002021c, 0x00020200,
  58	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  59	0x340c, 0x000000c0, 0x00800040,
  60	0x360c, 0x000000c0, 0x00800040,
  61	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  62	mmFBC_MISC, 0x00200000, 0x50100000,
  63	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  64	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  65	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  66	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  67	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  68	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  69	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  70	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  71	0x000c, 0xffffffff, 0x0040,
  72	0x000d, 0x00000040, 0x00004040,
  73	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  74	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  75	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  76	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
  77	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
  78	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
  79	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
  80	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
  81	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
  82	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
  83	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
  84	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
  85	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
  86	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  87	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  88	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
  89};
  90
  91static const u32 tahiti_golden_registers2[] =
  92{
  93	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
  94};
  95
  96static const u32 tahiti_golden_rlc_registers[] =
  97{
  98	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
  99	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 100	0x311f, 0xffffffff, 0x10104040,
 101	0x3122, 0xffffffff, 0x0100000a,
 102	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 103	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 104	mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 105};
 106
 107static const u32 pitcairn_golden_registers[] =
 108{
 109	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 110	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 111	mmDB_DEBUG, 0xffffffff, 0x00000000,
 112	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 113	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 114	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 115	0x340c, 0x000300c0, 0x00800040,
 116	0x360c, 0x000300c0, 0x00800040,
 117	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 118	mmFBC_MISC, 0x00200000, 0x50100000,
 119	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 120	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 121	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 122	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 123	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 124	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 125	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 126	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 127	0x000c, 0xffffffff, 0x0040,
 128	0x000d, 0x00000040, 0x00004040,
 129	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 130	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 131	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 132	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 133	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 134	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 135	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 136	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 137	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 138	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 139	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 140	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 141};
 142
 143static const u32 pitcairn_golden_rlc_registers[] =
 144{
 145	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 146	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 147	0x311f, 0xffffffff, 0x10102020,
 148	0x3122, 0xffffffff, 0x01000020,
 149	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 150	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 151};
 152
 153static const u32 verde_pg_init[] =
 154{
 155	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 156	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 157	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 158	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 159	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 160	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 161	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 162	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 163	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 164	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 165	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 166	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 167	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 168	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 169	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 170	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 171	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 172	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 173	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 174	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 175	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 176	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 177	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 178	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 179	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 180	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 181	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 182	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 183	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 184	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 185	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 186	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 187	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 188	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 189	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 190	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 191	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 192	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 193	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 194	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 195	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 196	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 198	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 199	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 200	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 201	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 202	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 203	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 204	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 205	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 206	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 207	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 208	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 209	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 210	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 211	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 212	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 213	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 214	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 215	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 216	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 217	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 218	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 219	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 220	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 221	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 222	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 223	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 224	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 225	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 226	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 227	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 228	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 229	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 230	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 231	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 232	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 233	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 234	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 235	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 236	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 237	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 238	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 239	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 240	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 241	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 242	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 243	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 244	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 245	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 246	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 247	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 248	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 249	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 250	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 251	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 252	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 253	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 254	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 255	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 256	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 257	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 258	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 259	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 260	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 261	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 262	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 263	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 264	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 265	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 266	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 267	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 268	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 269	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 270	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 271	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 272	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 273	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 274	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 275	mmGMCON_MISC2, 0xfc00, 0x2000,
 276	mmGMCON_MISC3, 0xffffffff, 0xfc0,
 277	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 278};
 279
 280static const u32 verde_golden_rlc_registers[] =
 281{
 282	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 283	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 284	0x311f, 0xffffffff, 0x10808020,
 285	0x3122, 0xffffffff, 0x00800008,
 286	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 287	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 288};
 289
 290static const u32 verde_golden_registers[] =
 291{
 292	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 293	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 294	mmDB_DEBUG, 0xffffffff, 0x00000000,
 295	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 296	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 297	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 298	0x340c, 0x000300c0, 0x00800040,
 299	0x360c, 0x000300c0, 0x00800040,
 300	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 301	mmFBC_MISC, 0x00200000, 0x50100000,
 302	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 303	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 304	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 305	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 306	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 307	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 308	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 309	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 310	0x000c, 0xffffffff, 0x0040,
 311	0x000d, 0x00000040, 0x00004040,
 312	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 313	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 314	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 315	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 316	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 317	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 318	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 319	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 320	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 321	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 322	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 323	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 324	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 325	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 326};
 327
 328static const u32 oland_golden_registers[] =
 329{
 330	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 331	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 332	mmDB_DEBUG, 0xffffffff, 0x00000000,
 333	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 334	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 335	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 336	0x340c, 0x000300c0, 0x00800040,
 337	0x360c, 0x000300c0, 0x00800040,
 338	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 339	mmFBC_MISC, 0x00200000, 0x50100000,
 340	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 341	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 342	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 343	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 344	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 345	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 346	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 347	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 348	0x000c, 0xffffffff, 0x0040,
 349	0x000d, 0x00000040, 0x00004040,
 350	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 351	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 352	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 353	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 354	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 355	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 356	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 357	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 358	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 359	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 360	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 361	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 362
 363};
 364
 365static const u32 oland_golden_rlc_registers[] =
 366{
 367	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 368	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 369	0x311f, 0xffffffff, 0x10104040,
 370	0x3122, 0xffffffff, 0x0100000a,
 371	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 372	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 373};
 374
 375static const u32 hainan_golden_registers[] =
 376{
 377	0x17bc, 0x00000030, 0x00000011,
 378	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 379	mmDB_DEBUG, 0xffffffff, 0x00000000,
 380	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 381	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 382	0x031e, 0x00000080, 0x00000000,
 383	0x3430, 0xff000fff, 0x00000100,
 384	0x340c, 0x000300c0, 0x00800040,
 385	0x3630, 0xff000fff, 0x00000100,
 386	0x360c, 0x000300c0, 0x00800040,
 387	0x16ec, 0x000000f0, 0x00000070,
 388	0x16f0, 0x00200000, 0x50100000,
 389	0x1c0c, 0x31000311, 0x00000011,
 390	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 391	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 392	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 393	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 394	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 395	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 396	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 397	0x000c, 0xffffffff, 0x0040,
 398	0x000d, 0x00000040, 0x00004040,
 399	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 400	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 401	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 402	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 403	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 404	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 405	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 406	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 407	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 408	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 409	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 410	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 411};
 412
 413static const u32 hainan_golden_registers2[] =
 414{
 415	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 416};
 417
 418static const u32 tahiti_mgcg_cgcg_init[] =
 419{
 420	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 421	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 422	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 423	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 424	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 425	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 426	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 427	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 428	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 429	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 430	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 431	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 432	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 433	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 434	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 435	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 436	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 437	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 438	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 439	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 440	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 441	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 442	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 443	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 444	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 445	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 446	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 447	0x2458, 0xffffffff, 0x00010000,
 448	0x2459, 0xffffffff, 0x00030002,
 449	0x245a, 0xffffffff, 0x00040007,
 450	0x245b, 0xffffffff, 0x00060005,
 451	0x245c, 0xffffffff, 0x00090008,
 452	0x245d, 0xffffffff, 0x00020001,
 453	0x245e, 0xffffffff, 0x00040003,
 454	0x245f, 0xffffffff, 0x00000007,
 455	0x2460, 0xffffffff, 0x00060005,
 456	0x2461, 0xffffffff, 0x00090008,
 457	0x2462, 0xffffffff, 0x00030002,
 458	0x2463, 0xffffffff, 0x00050004,
 459	0x2464, 0xffffffff, 0x00000008,
 460	0x2465, 0xffffffff, 0x00070006,
 461	0x2466, 0xffffffff, 0x000a0009,
 462	0x2467, 0xffffffff, 0x00040003,
 463	0x2468, 0xffffffff, 0x00060005,
 464	0x2469, 0xffffffff, 0x00000009,
 465	0x246a, 0xffffffff, 0x00080007,
 466	0x246b, 0xffffffff, 0x000b000a,
 467	0x246c, 0xffffffff, 0x00050004,
 468	0x246d, 0xffffffff, 0x00070006,
 469	0x246e, 0xffffffff, 0x0008000b,
 470	0x246f, 0xffffffff, 0x000a0009,
 471	0x2470, 0xffffffff, 0x000d000c,
 472	0x2471, 0xffffffff, 0x00060005,
 473	0x2472, 0xffffffff, 0x00080007,
 474	0x2473, 0xffffffff, 0x0000000b,
 475	0x2474, 0xffffffff, 0x000a0009,
 476	0x2475, 0xffffffff, 0x000d000c,
 477	0x2476, 0xffffffff, 0x00070006,
 478	0x2477, 0xffffffff, 0x00090008,
 479	0x2478, 0xffffffff, 0x0000000c,
 480	0x2479, 0xffffffff, 0x000b000a,
 481	0x247a, 0xffffffff, 0x000e000d,
 482	0x247b, 0xffffffff, 0x00080007,
 483	0x247c, 0xffffffff, 0x000a0009,
 484	0x247d, 0xffffffff, 0x0000000d,
 485	0x247e, 0xffffffff, 0x000c000b,
 486	0x247f, 0xffffffff, 0x000f000e,
 487	0x2480, 0xffffffff, 0x00090008,
 488	0x2481, 0xffffffff, 0x000b000a,
 489	0x2482, 0xffffffff, 0x000c000f,
 490	0x2483, 0xffffffff, 0x000e000d,
 491	0x2484, 0xffffffff, 0x00110010,
 492	0x2485, 0xffffffff, 0x000a0009,
 493	0x2486, 0xffffffff, 0x000c000b,
 494	0x2487, 0xffffffff, 0x0000000f,
 495	0x2488, 0xffffffff, 0x000e000d,
 496	0x2489, 0xffffffff, 0x00110010,
 497	0x248a, 0xffffffff, 0x000b000a,
 498	0x248b, 0xffffffff, 0x000d000c,
 499	0x248c, 0xffffffff, 0x00000010,
 500	0x248d, 0xffffffff, 0x000f000e,
 501	0x248e, 0xffffffff, 0x00120011,
 502	0x248f, 0xffffffff, 0x000c000b,
 503	0x2490, 0xffffffff, 0x000e000d,
 504	0x2491, 0xffffffff, 0x00000011,
 505	0x2492, 0xffffffff, 0x0010000f,
 506	0x2493, 0xffffffff, 0x00130012,
 507	0x2494, 0xffffffff, 0x000d000c,
 508	0x2495, 0xffffffff, 0x000f000e,
 509	0x2496, 0xffffffff, 0x00100013,
 510	0x2497, 0xffffffff, 0x00120011,
 511	0x2498, 0xffffffff, 0x00150014,
 512	0x2499, 0xffffffff, 0x000e000d,
 513	0x249a, 0xffffffff, 0x0010000f,
 514	0x249b, 0xffffffff, 0x00000013,
 515	0x249c, 0xffffffff, 0x00120011,
 516	0x249d, 0xffffffff, 0x00150014,
 517	0x249e, 0xffffffff, 0x000f000e,
 518	0x249f, 0xffffffff, 0x00110010,
 519	0x24a0, 0xffffffff, 0x00000014,
 520	0x24a1, 0xffffffff, 0x00130012,
 521	0x24a2, 0xffffffff, 0x00160015,
 522	0x24a3, 0xffffffff, 0x0010000f,
 523	0x24a4, 0xffffffff, 0x00120011,
 524	0x24a5, 0xffffffff, 0x00000015,
 525	0x24a6, 0xffffffff, 0x00140013,
 526	0x24a7, 0xffffffff, 0x00170016,
 527	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 528	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 529	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 530	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 531	0x000c, 0xffffffff, 0x0000001c,
 532	0x000d, 0x000f0000, 0x000f0000,
 533	0x0583, 0xffffffff, 0x00000100,
 534	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 535	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 536	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 537	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 538	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 539	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 540	0x157a, 0x00000001, 0x00000001,
 541	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 542	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 543	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 544	0x3430, 0xfffffff0, 0x00000100,
 545	0x3630, 0xfffffff0, 0x00000100,
 546};
 547static const u32 pitcairn_mgcg_cgcg_init[] =
 548{
 549	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 550	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 551	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 552	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 553	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 554	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 555	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 556	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 557	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 558	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 559	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 560	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 561	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 562	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 563	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 564	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 565	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 566	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 567	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 568	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 569	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 570	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 571	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 572	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 573	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 574	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 575	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 576	0x2458, 0xffffffff, 0x00010000,
 577	0x2459, 0xffffffff, 0x00030002,
 578	0x245a, 0xffffffff, 0x00040007,
 579	0x245b, 0xffffffff, 0x00060005,
 580	0x245c, 0xffffffff, 0x00090008,
 581	0x245d, 0xffffffff, 0x00020001,
 582	0x245e, 0xffffffff, 0x00040003,
 583	0x245f, 0xffffffff, 0x00000007,
 584	0x2460, 0xffffffff, 0x00060005,
 585	0x2461, 0xffffffff, 0x00090008,
 586	0x2462, 0xffffffff, 0x00030002,
 587	0x2463, 0xffffffff, 0x00050004,
 588	0x2464, 0xffffffff, 0x00000008,
 589	0x2465, 0xffffffff, 0x00070006,
 590	0x2466, 0xffffffff, 0x000a0009,
 591	0x2467, 0xffffffff, 0x00040003,
 592	0x2468, 0xffffffff, 0x00060005,
 593	0x2469, 0xffffffff, 0x00000009,
 594	0x246a, 0xffffffff, 0x00080007,
 595	0x246b, 0xffffffff, 0x000b000a,
 596	0x246c, 0xffffffff, 0x00050004,
 597	0x246d, 0xffffffff, 0x00070006,
 598	0x246e, 0xffffffff, 0x0008000b,
 599	0x246f, 0xffffffff, 0x000a0009,
 600	0x2470, 0xffffffff, 0x000d000c,
 601	0x2480, 0xffffffff, 0x00090008,
 602	0x2481, 0xffffffff, 0x000b000a,
 603	0x2482, 0xffffffff, 0x000c000f,
 604	0x2483, 0xffffffff, 0x000e000d,
 605	0x2484, 0xffffffff, 0x00110010,
 606	0x2485, 0xffffffff, 0x000a0009,
 607	0x2486, 0xffffffff, 0x000c000b,
 608	0x2487, 0xffffffff, 0x0000000f,
 609	0x2488, 0xffffffff, 0x000e000d,
 610	0x2489, 0xffffffff, 0x00110010,
 611	0x248a, 0xffffffff, 0x000b000a,
 612	0x248b, 0xffffffff, 0x000d000c,
 613	0x248c, 0xffffffff, 0x00000010,
 614	0x248d, 0xffffffff, 0x000f000e,
 615	0x248e, 0xffffffff, 0x00120011,
 616	0x248f, 0xffffffff, 0x000c000b,
 617	0x2490, 0xffffffff, 0x000e000d,
 618	0x2491, 0xffffffff, 0x00000011,
 619	0x2492, 0xffffffff, 0x0010000f,
 620	0x2493, 0xffffffff, 0x00130012,
 621	0x2494, 0xffffffff, 0x000d000c,
 622	0x2495, 0xffffffff, 0x000f000e,
 623	0x2496, 0xffffffff, 0x00100013,
 624	0x2497, 0xffffffff, 0x00120011,
 625	0x2498, 0xffffffff, 0x00150014,
 626	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 627	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 628	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 629	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 630	0x000c, 0xffffffff, 0x0000001c,
 631	0x000d, 0x000f0000, 0x000f0000,
 632	0x0583, 0xffffffff, 0x00000100,
 633	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 634	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 635	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 636	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 637	0x157a, 0x00000001, 0x00000001,
 638	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 639	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 640	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 641	0x3430, 0xfffffff0, 0x00000100,
 642	0x3630, 0xfffffff0, 0x00000100,
 643};
 644
 645static const u32 verde_mgcg_cgcg_init[] =
 646{
 647	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 648	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 649	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 650	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 651	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 652	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 653	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 654	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 655	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 656	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 657	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 658	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 659	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 660	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 661	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 662	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 663	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 664	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 665	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 666	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 667	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 668	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 669	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 670	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 671	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 672	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 673	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 674	0x2458, 0xffffffff, 0x00010000,
 675	0x2459, 0xffffffff, 0x00030002,
 676	0x245a, 0xffffffff, 0x00040007,
 677	0x245b, 0xffffffff, 0x00060005,
 678	0x245c, 0xffffffff, 0x00090008,
 679	0x245d, 0xffffffff, 0x00020001,
 680	0x245e, 0xffffffff, 0x00040003,
 681	0x245f, 0xffffffff, 0x00000007,
 682	0x2460, 0xffffffff, 0x00060005,
 683	0x2461, 0xffffffff, 0x00090008,
 684	0x2462, 0xffffffff, 0x00030002,
 685	0x2463, 0xffffffff, 0x00050004,
 686	0x2464, 0xffffffff, 0x00000008,
 687	0x2465, 0xffffffff, 0x00070006,
 688	0x2466, 0xffffffff, 0x000a0009,
 689	0x2467, 0xffffffff, 0x00040003,
 690	0x2468, 0xffffffff, 0x00060005,
 691	0x2469, 0xffffffff, 0x00000009,
 692	0x246a, 0xffffffff, 0x00080007,
 693	0x246b, 0xffffffff, 0x000b000a,
 694	0x246c, 0xffffffff, 0x00050004,
 695	0x246d, 0xffffffff, 0x00070006,
 696	0x246e, 0xffffffff, 0x0008000b,
 697	0x246f, 0xffffffff, 0x000a0009,
 698	0x2470, 0xffffffff, 0x000d000c,
 699	0x2480, 0xffffffff, 0x00090008,
 700	0x2481, 0xffffffff, 0x000b000a,
 701	0x2482, 0xffffffff, 0x000c000f,
 702	0x2483, 0xffffffff, 0x000e000d,
 703	0x2484, 0xffffffff, 0x00110010,
 704	0x2485, 0xffffffff, 0x000a0009,
 705	0x2486, 0xffffffff, 0x000c000b,
 706	0x2487, 0xffffffff, 0x0000000f,
 707	0x2488, 0xffffffff, 0x000e000d,
 708	0x2489, 0xffffffff, 0x00110010,
 709	0x248a, 0xffffffff, 0x000b000a,
 710	0x248b, 0xffffffff, 0x000d000c,
 711	0x248c, 0xffffffff, 0x00000010,
 712	0x248d, 0xffffffff, 0x000f000e,
 713	0x248e, 0xffffffff, 0x00120011,
 714	0x248f, 0xffffffff, 0x000c000b,
 715	0x2490, 0xffffffff, 0x000e000d,
 716	0x2491, 0xffffffff, 0x00000011,
 717	0x2492, 0xffffffff, 0x0010000f,
 718	0x2493, 0xffffffff, 0x00130012,
 719	0x2494, 0xffffffff, 0x000d000c,
 720	0x2495, 0xffffffff, 0x000f000e,
 721	0x2496, 0xffffffff, 0x00100013,
 722	0x2497, 0xffffffff, 0x00120011,
 723	0x2498, 0xffffffff, 0x00150014,
 724	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 725	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 726	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 727	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 728	0x000c, 0xffffffff, 0x0000001c,
 729	0x000d, 0x000f0000, 0x000f0000,
 730	0x0583, 0xffffffff, 0x00000100,
 731	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 732	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 733	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 734	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 735	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 736	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 737	0x157a, 0x00000001, 0x00000001,
 738	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 739	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 740	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 741	0x3430, 0xfffffff0, 0x00000100,
 742	0x3630, 0xfffffff0, 0x00000100,
 743};
 744
 745static const u32 oland_mgcg_cgcg_init[] =
 746{
 747	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 748	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 749	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 750	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 751	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 752	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 753	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 754	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 755	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 756	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 757	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 758	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 759	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 760	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 761	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 762	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 763	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 764	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 765	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 766	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 767	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 768	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 769	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 770	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 771	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 772	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 773	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 774	0x2458, 0xffffffff, 0x00010000,
 775	0x2459, 0xffffffff, 0x00030002,
 776	0x245a, 0xffffffff, 0x00040007,
 777	0x245b, 0xffffffff, 0x00060005,
 778	0x245c, 0xffffffff, 0x00090008,
 779	0x245d, 0xffffffff, 0x00020001,
 780	0x245e, 0xffffffff, 0x00040003,
 781	0x245f, 0xffffffff, 0x00000007,
 782	0x2460, 0xffffffff, 0x00060005,
 783	0x2461, 0xffffffff, 0x00090008,
 784	0x2462, 0xffffffff, 0x00030002,
 785	0x2463, 0xffffffff, 0x00050004,
 786	0x2464, 0xffffffff, 0x00000008,
 787	0x2465, 0xffffffff, 0x00070006,
 788	0x2466, 0xffffffff, 0x000a0009,
 789	0x2467, 0xffffffff, 0x00040003,
 790	0x2468, 0xffffffff, 0x00060005,
 791	0x2469, 0xffffffff, 0x00000009,
 792	0x246a, 0xffffffff, 0x00080007,
 793	0x246b, 0xffffffff, 0x000b000a,
 794	0x246c, 0xffffffff, 0x00050004,
 795	0x246d, 0xffffffff, 0x00070006,
 796	0x246e, 0xffffffff, 0x0008000b,
 797	0x246f, 0xffffffff, 0x000a0009,
 798	0x2470, 0xffffffff, 0x000d000c,
 799	0x2471, 0xffffffff, 0x00060005,
 800	0x2472, 0xffffffff, 0x00080007,
 801	0x2473, 0xffffffff, 0x0000000b,
 802	0x2474, 0xffffffff, 0x000a0009,
 803	0x2475, 0xffffffff, 0x000d000c,
 804	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 805	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 806	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 807	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 808	0x000c, 0xffffffff, 0x0000001c,
 809	0x000d, 0x000f0000, 0x000f0000,
 810	0x0583, 0xffffffff, 0x00000100,
 811	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 812	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 813	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 814	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 815	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 816	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 817	0x157a, 0x00000001, 0x00000001,
 818	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 819	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 820	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 821	0x3430, 0xfffffff0, 0x00000100,
 822	0x3630, 0xfffffff0, 0x00000100,
 823};
 824
 825static const u32 hainan_mgcg_cgcg_init[] =
 826{
 827	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 828	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 829	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 830	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 831	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 832	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 833	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 834	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 835	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 836	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 837	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 838	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 839	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 840	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 841	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 842	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 843	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 844	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 845	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 846	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 847	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 848	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 849	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 850	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 851	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 852	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 853	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 854	0x2458, 0xffffffff, 0x00010000,
 855	0x2459, 0xffffffff, 0x00030002,
 856	0x245a, 0xffffffff, 0x00040007,
 857	0x245b, 0xffffffff, 0x00060005,
 858	0x245c, 0xffffffff, 0x00090008,
 859	0x245d, 0xffffffff, 0x00020001,
 860	0x245e, 0xffffffff, 0x00040003,
 861	0x245f, 0xffffffff, 0x00000007,
 862	0x2460, 0xffffffff, 0x00060005,
 863	0x2461, 0xffffffff, 0x00090008,
 864	0x2462, 0xffffffff, 0x00030002,
 865	0x2463, 0xffffffff, 0x00050004,
 866	0x2464, 0xffffffff, 0x00000008,
 867	0x2465, 0xffffffff, 0x00070006,
 868	0x2466, 0xffffffff, 0x000a0009,
 869	0x2467, 0xffffffff, 0x00040003,
 870	0x2468, 0xffffffff, 0x00060005,
 871	0x2469, 0xffffffff, 0x00000009,
 872	0x246a, 0xffffffff, 0x00080007,
 873	0x246b, 0xffffffff, 0x000b000a,
 874	0x246c, 0xffffffff, 0x00050004,
 875	0x246d, 0xffffffff, 0x00070006,
 876	0x246e, 0xffffffff, 0x0008000b,
 877	0x246f, 0xffffffff, 0x000a0009,
 878	0x2470, 0xffffffff, 0x000d000c,
 879	0x2471, 0xffffffff, 0x00060005,
 880	0x2472, 0xffffffff, 0x00080007,
 881	0x2473, 0xffffffff, 0x0000000b,
 882	0x2474, 0xffffffff, 0x000a0009,
 883	0x2475, 0xffffffff, 0x000d000c,
 884	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 885	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 886	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 887	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 888	0x000c, 0xffffffff, 0x0000001c,
 889	0x000d, 0x000f0000, 0x000f0000,
 890	0x0583, 0xffffffff, 0x00000100,
 891	0x0409, 0xffffffff, 0x00000100,
 892	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 893	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 894	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 895	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 896	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 897	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 898	0x3430, 0xfffffff0, 0x00000100,
 899	0x3630, 0xfffffff0, 0x00000100,
 900};
 901
 902static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 903{
 904	unsigned long flags;
 905	u32 r;
 906
 907	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 908	WREG32(AMDGPU_PCIE_INDEX, reg);
 909	(void)RREG32(AMDGPU_PCIE_INDEX);
 910	r = RREG32(AMDGPU_PCIE_DATA);
 911	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 912	return r;
 913}
 914
 915static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 916{
 917	unsigned long flags;
 918
 919	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 920	WREG32(AMDGPU_PCIE_INDEX, reg);
 921	(void)RREG32(AMDGPU_PCIE_INDEX);
 922	WREG32(AMDGPU_PCIE_DATA, v);
 923	(void)RREG32(AMDGPU_PCIE_DATA);
 924	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 925}
 926
 927static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 928{
 929	unsigned long flags;
 930	u32 r;
 931
 932	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 933	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 934	(void)RREG32(PCIE_PORT_INDEX);
 935	r = RREG32(PCIE_PORT_DATA);
 936	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 937	return r;
 938}
 939
 940static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 941{
 942	unsigned long flags;
 943
 944	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 945	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
 946	(void)RREG32(PCIE_PORT_INDEX);
 947	WREG32(PCIE_PORT_DATA, (v));
 948	(void)RREG32(PCIE_PORT_DATA);
 949	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 950}
 951
 952static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
 953{
 954	unsigned long flags;
 955	u32 r;
 956
 957	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 958	WREG32(SMC_IND_INDEX_0, (reg));
 959	r = RREG32(SMC_IND_DATA_0);
 960	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961	return r;
 962}
 963
 964static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 965{
 966	unsigned long flags;
 967
 968	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 969	WREG32(SMC_IND_INDEX_0, (reg));
 970	WREG32(SMC_IND_DATA_0, (v));
 971	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 972}
 973
 974static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
 975	{GRBM_STATUS},
 976	{GB_ADDR_CONFIG},
 977	{MC_ARB_RAMCFG},
 978	{GB_TILE_MODE0},
 979	{GB_TILE_MODE1},
 980	{GB_TILE_MODE2},
 981	{GB_TILE_MODE3},
 982	{GB_TILE_MODE4},
 983	{GB_TILE_MODE5},
 984	{GB_TILE_MODE6},
 985	{GB_TILE_MODE7},
 986	{GB_TILE_MODE8},
 987	{GB_TILE_MODE9},
 988	{GB_TILE_MODE10},
 989	{GB_TILE_MODE11},
 990	{GB_TILE_MODE12},
 991	{GB_TILE_MODE13},
 992	{GB_TILE_MODE14},
 993	{GB_TILE_MODE15},
 994	{GB_TILE_MODE16},
 995	{GB_TILE_MODE17},
 996	{GB_TILE_MODE18},
 997	{GB_TILE_MODE19},
 998	{GB_TILE_MODE20},
 999	{GB_TILE_MODE21},
1000	{GB_TILE_MODE22},
1001	{GB_TILE_MODE23},
1002	{GB_TILE_MODE24},
1003	{GB_TILE_MODE25},
1004	{GB_TILE_MODE26},
1005	{GB_TILE_MODE27},
1006	{GB_TILE_MODE28},
1007	{GB_TILE_MODE29},
1008	{GB_TILE_MODE30},
1009	{GB_TILE_MODE31},
1010	{CC_RB_BACKEND_DISABLE, true},
1011	{GC_USER_RB_BACKEND_DISABLE, true},
1012	{PA_SC_RASTER_CONFIG, true},
1013};
1014
1015static uint32_t si_get_register_value(struct amdgpu_device *adev,
1016				      bool indexed, u32 se_num,
1017				      u32 sh_num, u32 reg_offset)
1018{
1019	if (indexed) {
1020		uint32_t val;
1021		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1022		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1023
1024		switch (reg_offset) {
1025		case mmCC_RB_BACKEND_DISABLE:
1026			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1027		case mmGC_USER_RB_BACKEND_DISABLE:
1028			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1029		case mmPA_SC_RASTER_CONFIG:
1030			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1031		}
1032
1033		mutex_lock(&adev->grbm_idx_mutex);
1034		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1035			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1036
1037		val = RREG32(reg_offset);
1038
1039		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1040			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1041		mutex_unlock(&adev->grbm_idx_mutex);
1042		return val;
1043	} else {
1044		unsigned idx;
1045
1046		switch (reg_offset) {
1047		case mmGB_ADDR_CONFIG:
1048			return adev->gfx.config.gb_addr_config;
1049		case mmMC_ARB_RAMCFG:
1050			return adev->gfx.config.mc_arb_ramcfg;
1051		case mmGB_TILE_MODE0:
1052		case mmGB_TILE_MODE1:
1053		case mmGB_TILE_MODE2:
1054		case mmGB_TILE_MODE3:
1055		case mmGB_TILE_MODE4:
1056		case mmGB_TILE_MODE5:
1057		case mmGB_TILE_MODE6:
1058		case mmGB_TILE_MODE7:
1059		case mmGB_TILE_MODE8:
1060		case mmGB_TILE_MODE9:
1061		case mmGB_TILE_MODE10:
1062		case mmGB_TILE_MODE11:
1063		case mmGB_TILE_MODE12:
1064		case mmGB_TILE_MODE13:
1065		case mmGB_TILE_MODE14:
1066		case mmGB_TILE_MODE15:
1067		case mmGB_TILE_MODE16:
1068		case mmGB_TILE_MODE17:
1069		case mmGB_TILE_MODE18:
1070		case mmGB_TILE_MODE19:
1071		case mmGB_TILE_MODE20:
1072		case mmGB_TILE_MODE21:
1073		case mmGB_TILE_MODE22:
1074		case mmGB_TILE_MODE23:
1075		case mmGB_TILE_MODE24:
1076		case mmGB_TILE_MODE25:
1077		case mmGB_TILE_MODE26:
1078		case mmGB_TILE_MODE27:
1079		case mmGB_TILE_MODE28:
1080		case mmGB_TILE_MODE29:
1081		case mmGB_TILE_MODE30:
1082		case mmGB_TILE_MODE31:
1083			idx = (reg_offset - mmGB_TILE_MODE0);
1084			return adev->gfx.config.tile_mode_array[idx];
1085		default:
1086			return RREG32(reg_offset);
1087		}
1088	}
1089}
 
1090static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1091			     u32 sh_num, u32 reg_offset, u32 *value)
1092{
1093	uint32_t i;
1094
1095	*value = 0;
1096	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1097		bool indexed = si_allowed_read_registers[i].grbm_indexed;
1098
1099		if (reg_offset != si_allowed_read_registers[i].reg_offset)
1100			continue;
1101
1102		*value = si_get_register_value(adev, indexed, se_num, sh_num,
1103					       reg_offset);
 
 
 
1104		return 0;
1105	}
1106	return -EINVAL;
1107}
1108
1109static bool si_read_disabled_bios(struct amdgpu_device *adev)
1110{
1111	u32 bus_cntl;
1112	u32 d1vga_control = 0;
1113	u32 d2vga_control = 0;
1114	u32 vga_render_control = 0;
1115	u32 rom_cntl;
1116	bool r;
1117
1118	bus_cntl = RREG32(R600_BUS_CNTL);
1119	if (adev->mode_info.num_crtc) {
1120		d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1121		d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1122		vga_render_control = RREG32(VGA_RENDER_CONTROL);
1123	}
1124	rom_cntl = RREG32(R600_ROM_CNTL);
1125
1126	/* enable the rom */
1127	WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1128	if (adev->mode_info.num_crtc) {
1129		/* Disable VGA mode */
1130		WREG32(AVIVO_D1VGA_CONTROL,
1131		       (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1132					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1133		WREG32(AVIVO_D2VGA_CONTROL,
1134		       (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1135					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1136		WREG32(VGA_RENDER_CONTROL,
1137		       (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1138	}
1139	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1140
1141	r = amdgpu_read_bios(adev);
1142
1143	/* restore regs */
1144	WREG32(R600_BUS_CNTL, bus_cntl);
1145	if (adev->mode_info.num_crtc) {
1146		WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1147		WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1148		WREG32(VGA_RENDER_CONTROL, vga_render_control);
1149	}
1150	WREG32(R600_ROM_CNTL, rom_cntl);
1151	return r;
1152}
1153
1154#define mmROM_INDEX 0x2A
1155#define mmROM_DATA  0x2B
1156
1157static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1158				  u8 *bios, u32 length_bytes)
1159{
1160	u32 *dw_ptr;
1161	u32 i, length_dw;
1162
1163	if (bios == NULL)
1164		return false;
1165	if (length_bytes == 0)
1166		return false;
1167	/* APU vbios image is part of sbios image */
1168	if (adev->flags & AMD_IS_APU)
1169		return false;
1170
1171	dw_ptr = (u32 *)bios;
1172	length_dw = ALIGN(length_bytes, 4) / 4;
1173	/* set rom index to 0 */
1174	WREG32(mmROM_INDEX, 0);
1175	for (i = 0; i < length_dw; i++)
1176		dw_ptr[i] = RREG32(mmROM_DATA);
1177
1178	return true;
1179}
1180
1181//xxx: not implemented
1182static int si_asic_reset(struct amdgpu_device *adev)
1183{
1184	return 0;
1185}
1186
1187static u32 si_get_config_memsize(struct amdgpu_device *adev)
1188{
1189	return RREG32(mmCONFIG_MEMSIZE);
1190}
1191
1192static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1193{
1194	uint32_t temp;
1195
1196	temp = RREG32(CONFIG_CNTL);
1197	if (state == false) {
1198		temp &= ~(1<<0);
1199		temp |= (1<<1);
1200	} else {
1201		temp &= ~(1<<1);
1202	}
1203	WREG32(CONFIG_CNTL, temp);
1204}
1205
1206static u32 si_get_xclk(struct amdgpu_device *adev)
1207{
1208        u32 reference_clock = adev->clock.spll.reference_freq;
1209	u32 tmp;
1210
1211	tmp = RREG32(CG_CLKPIN_CNTL_2);
1212	if (tmp & MUX_TCLK_TO_XCLK)
1213		return TCLK;
1214
1215	tmp = RREG32(CG_CLKPIN_CNTL);
1216	if (tmp & XTALIN_DIVIDE)
1217		return reference_clock / 4;
1218
1219	return reference_clock;
1220}
1221
1222//xxx:not implemented
1223static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1224{
1225	return 0;
1226}
1227
1228static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1229{
1230	if (is_virtual_machine()) /* passthrough mode */
1231		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1232}
1233
1234static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1235{
1236	if (!ring || !ring->funcs->emit_wreg) {
1237		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1238		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1239	} else {
1240		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1241	}
1242}
1243
1244static void si_invalidate_hdp(struct amdgpu_device *adev,
1245			      struct amdgpu_ring *ring)
1246{
1247	if (!ring || !ring->funcs->emit_wreg) {
1248		WREG32(mmHDP_DEBUG0, 1);
1249		RREG32(mmHDP_DEBUG0);
1250	} else {
1251		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1252	}
1253}
1254
1255static int si_get_pcie_lanes(struct amdgpu_device *adev)
1256{
1257	u32 link_width_cntl;
1258
1259	if (adev->flags & AMD_IS_APU)
1260		return 0;
1261
1262	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1263
1264	switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {
1265	case LC_LINK_WIDTH_X1:
1266		return 1;
1267	case LC_LINK_WIDTH_X2:
1268		return 2;
1269	case LC_LINK_WIDTH_X4:
1270		return 4;
1271	case LC_LINK_WIDTH_X8:
1272		return 8;
1273	case LC_LINK_WIDTH_X0:
1274	case LC_LINK_WIDTH_X16:
1275	default:
1276		return 16;
1277	}
1278}
1279
1280static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1281{
1282	u32 link_width_cntl, mask;
1283
1284	if (adev->flags & AMD_IS_APU)
1285		return;
1286
1287	switch (lanes) {
1288	case 0:
1289		mask = LC_LINK_WIDTH_X0;
1290		break;
1291	case 1:
1292		mask = LC_LINK_WIDTH_X1;
1293		break;
1294	case 2:
1295		mask = LC_LINK_WIDTH_X2;
1296		break;
1297	case 4:
1298		mask = LC_LINK_WIDTH_X4;
1299		break;
1300	case 8:
1301		mask = LC_LINK_WIDTH_X8;
1302		break;
1303	case 16:
1304		mask = LC_LINK_WIDTH_X16;
1305		break;
1306	default:
1307		DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1308		return;
1309	}
1310
1311	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1312	link_width_cntl &= ~LC_LINK_WIDTH_MASK;
1313	link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;
1314	link_width_cntl |= (LC_RECONFIG_NOW |
1315			    LC_RECONFIG_ARC_MISSING_ESCAPE);
1316
1317	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1318}
1319
1320static const struct amdgpu_asic_funcs si_asic_funcs =
1321{
1322	.read_disabled_bios = &si_read_disabled_bios,
1323	.read_bios_from_rom = &si_read_bios_from_rom,
1324	.read_register = &si_read_register,
1325	.reset = &si_asic_reset,
1326	.set_vga_state = &si_vga_set_state,
1327	.get_xclk = &si_get_xclk,
1328	.set_uvd_clocks = &si_set_uvd_clocks,
1329	.set_vce_clocks = NULL,
1330	.get_pcie_lanes = &si_get_pcie_lanes,
1331	.set_pcie_lanes = &si_set_pcie_lanes,
1332	.get_config_memsize = &si_get_config_memsize,
1333	.flush_hdp = &si_flush_hdp,
1334	.invalidate_hdp = &si_invalidate_hdp,
1335};
1336
1337static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1338{
1339	return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1340		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1341}
1342
1343static int si_common_early_init(void *handle)
1344{
1345	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1346
1347	adev->smc_rreg = &si_smc_rreg;
1348	adev->smc_wreg = &si_smc_wreg;
1349	adev->pcie_rreg = &si_pcie_rreg;
1350	adev->pcie_wreg = &si_pcie_wreg;
1351	adev->pciep_rreg = &si_pciep_rreg;
1352	adev->pciep_wreg = &si_pciep_wreg;
1353	adev->uvd_ctx_rreg = NULL;
1354	adev->uvd_ctx_wreg = NULL;
1355	adev->didt_rreg = NULL;
1356	adev->didt_wreg = NULL;
1357
1358	adev->asic_funcs = &si_asic_funcs;
1359
1360	adev->rev_id = si_get_rev_id(adev);
1361	adev->external_rev_id = 0xFF;
1362	switch (adev->asic_type) {
1363	case CHIP_TAHITI:
1364		adev->cg_flags =
1365			AMD_CG_SUPPORT_GFX_MGCG |
1366			AMD_CG_SUPPORT_GFX_MGLS |
1367			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1368			AMD_CG_SUPPORT_GFX_CGLS |
1369			AMD_CG_SUPPORT_GFX_CGTS |
1370			AMD_CG_SUPPORT_GFX_CP_LS |
1371			AMD_CG_SUPPORT_MC_MGCG |
1372			AMD_CG_SUPPORT_SDMA_MGCG |
1373			AMD_CG_SUPPORT_BIF_LS |
1374			AMD_CG_SUPPORT_VCE_MGCG |
1375			AMD_CG_SUPPORT_UVD_MGCG |
1376			AMD_CG_SUPPORT_HDP_LS |
1377			AMD_CG_SUPPORT_HDP_MGCG;
1378			adev->pg_flags = 0;
1379		adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1380					(adev->rev_id == 1) ? 5 : 6;
1381		break;
1382	case CHIP_PITCAIRN:
1383		adev->cg_flags =
1384			AMD_CG_SUPPORT_GFX_MGCG |
1385			AMD_CG_SUPPORT_GFX_MGLS |
1386			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1387			AMD_CG_SUPPORT_GFX_CGLS |
1388			AMD_CG_SUPPORT_GFX_CGTS |
1389			AMD_CG_SUPPORT_GFX_CP_LS |
1390			AMD_CG_SUPPORT_GFX_RLC_LS |
1391			AMD_CG_SUPPORT_MC_LS |
1392			AMD_CG_SUPPORT_MC_MGCG |
1393			AMD_CG_SUPPORT_SDMA_MGCG |
1394			AMD_CG_SUPPORT_BIF_LS |
1395			AMD_CG_SUPPORT_VCE_MGCG |
1396			AMD_CG_SUPPORT_UVD_MGCG |
1397			AMD_CG_SUPPORT_HDP_LS |
1398			AMD_CG_SUPPORT_HDP_MGCG;
1399		adev->pg_flags = 0;
1400		adev->external_rev_id = adev->rev_id + 20;
1401		break;
1402
1403	case CHIP_VERDE:
1404		adev->cg_flags =
1405			AMD_CG_SUPPORT_GFX_MGCG |
1406			AMD_CG_SUPPORT_GFX_MGLS |
1407			AMD_CG_SUPPORT_GFX_CGLS |
1408			AMD_CG_SUPPORT_GFX_CGTS |
1409			AMD_CG_SUPPORT_GFX_CGTS_LS |
1410			AMD_CG_SUPPORT_GFX_CP_LS |
1411			AMD_CG_SUPPORT_MC_LS |
1412			AMD_CG_SUPPORT_MC_MGCG |
1413			AMD_CG_SUPPORT_SDMA_MGCG |
1414			AMD_CG_SUPPORT_SDMA_LS |
1415			AMD_CG_SUPPORT_BIF_LS |
1416			AMD_CG_SUPPORT_VCE_MGCG |
1417			AMD_CG_SUPPORT_UVD_MGCG |
1418			AMD_CG_SUPPORT_HDP_LS |
1419			AMD_CG_SUPPORT_HDP_MGCG;
1420		adev->pg_flags = 0;
1421		//???
1422		adev->external_rev_id = adev->rev_id + 40;
1423		break;
1424	case CHIP_OLAND:
1425		adev->cg_flags =
1426			AMD_CG_SUPPORT_GFX_MGCG |
1427			AMD_CG_SUPPORT_GFX_MGLS |
1428			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1429			AMD_CG_SUPPORT_GFX_CGLS |
1430			AMD_CG_SUPPORT_GFX_CGTS |
1431			AMD_CG_SUPPORT_GFX_CP_LS |
1432			AMD_CG_SUPPORT_GFX_RLC_LS |
1433			AMD_CG_SUPPORT_MC_LS |
1434			AMD_CG_SUPPORT_MC_MGCG |
1435			AMD_CG_SUPPORT_SDMA_MGCG |
1436			AMD_CG_SUPPORT_BIF_LS |
1437			AMD_CG_SUPPORT_UVD_MGCG |
1438			AMD_CG_SUPPORT_HDP_LS |
1439			AMD_CG_SUPPORT_HDP_MGCG;
1440		adev->pg_flags = 0;
1441		adev->external_rev_id = 60;
1442		break;
1443	case CHIP_HAINAN:
1444		adev->cg_flags =
1445			AMD_CG_SUPPORT_GFX_MGCG |
1446			AMD_CG_SUPPORT_GFX_MGLS |
1447			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1448			AMD_CG_SUPPORT_GFX_CGLS |
1449			AMD_CG_SUPPORT_GFX_CGTS |
1450			AMD_CG_SUPPORT_GFX_CP_LS |
1451			AMD_CG_SUPPORT_GFX_RLC_LS |
1452			AMD_CG_SUPPORT_MC_LS |
1453			AMD_CG_SUPPORT_MC_MGCG |
1454			AMD_CG_SUPPORT_SDMA_MGCG |
1455			AMD_CG_SUPPORT_BIF_LS |
1456			AMD_CG_SUPPORT_HDP_LS |
1457			AMD_CG_SUPPORT_HDP_MGCG;
1458		adev->pg_flags = 0;
1459		adev->external_rev_id = 70;
1460		break;
1461
1462	default:
1463		return -EINVAL;
1464	}
1465
1466	return 0;
1467}
1468
1469static int si_common_sw_init(void *handle)
1470{
1471	return 0;
1472}
1473
1474static int si_common_sw_fini(void *handle)
1475{
1476	return 0;
1477}
1478
1479
1480static void si_init_golden_registers(struct amdgpu_device *adev)
1481{
1482	switch (adev->asic_type) {
1483	case CHIP_TAHITI:
1484		amdgpu_device_program_register_sequence(adev,
1485							tahiti_golden_registers,
1486							ARRAY_SIZE(tahiti_golden_registers));
1487		amdgpu_device_program_register_sequence(adev,
1488							tahiti_golden_rlc_registers,
1489							ARRAY_SIZE(tahiti_golden_rlc_registers));
1490		amdgpu_device_program_register_sequence(adev,
1491							tahiti_mgcg_cgcg_init,
1492							ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1493		amdgpu_device_program_register_sequence(adev,
1494							tahiti_golden_registers2,
1495							ARRAY_SIZE(tahiti_golden_registers2));
1496		break;
1497	case CHIP_PITCAIRN:
1498		amdgpu_device_program_register_sequence(adev,
1499							pitcairn_golden_registers,
1500							ARRAY_SIZE(pitcairn_golden_registers));
1501		amdgpu_device_program_register_sequence(adev,
1502							pitcairn_golden_rlc_registers,
1503							ARRAY_SIZE(pitcairn_golden_rlc_registers));
1504		amdgpu_device_program_register_sequence(adev,
1505							pitcairn_mgcg_cgcg_init,
1506							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1507		break;
1508	case CHIP_VERDE:
1509		amdgpu_device_program_register_sequence(adev,
1510							verde_golden_registers,
1511							ARRAY_SIZE(verde_golden_registers));
1512		amdgpu_device_program_register_sequence(adev,
1513							verde_golden_rlc_registers,
1514							ARRAY_SIZE(verde_golden_rlc_registers));
1515		amdgpu_device_program_register_sequence(adev,
1516							verde_mgcg_cgcg_init,
1517							ARRAY_SIZE(verde_mgcg_cgcg_init));
1518		amdgpu_device_program_register_sequence(adev,
1519							verde_pg_init,
1520							ARRAY_SIZE(verde_pg_init));
1521		break;
1522	case CHIP_OLAND:
1523		amdgpu_device_program_register_sequence(adev,
1524							oland_golden_registers,
1525							ARRAY_SIZE(oland_golden_registers));
1526		amdgpu_device_program_register_sequence(adev,
1527							oland_golden_rlc_registers,
1528							ARRAY_SIZE(oland_golden_rlc_registers));
1529		amdgpu_device_program_register_sequence(adev,
1530							oland_mgcg_cgcg_init,
1531							ARRAY_SIZE(oland_mgcg_cgcg_init));
1532		break;
1533	case CHIP_HAINAN:
1534		amdgpu_device_program_register_sequence(adev,
1535							hainan_golden_registers,
1536							ARRAY_SIZE(hainan_golden_registers));
1537		amdgpu_device_program_register_sequence(adev,
1538							hainan_golden_registers2,
1539							ARRAY_SIZE(hainan_golden_registers2));
1540		amdgpu_device_program_register_sequence(adev,
1541							hainan_mgcg_cgcg_init,
1542							ARRAY_SIZE(hainan_mgcg_cgcg_init));
1543		break;
1544
1545
1546	default:
1547		BUG();
1548	}
1549}
1550
1551static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1552{
1553	struct pci_dev *root = adev->pdev->bus->self;
1554	int bridge_pos, gpu_pos;
1555	u32 speed_cntl, current_data_rate;
1556	int i;
1557	u16 tmp16;
1558
1559	if (pci_is_root_bus(adev->pdev->bus))
1560		return;
1561
1562	if (amdgpu_pcie_gen2 == 0)
1563		return;
1564
1565	if (adev->flags & AMD_IS_APU)
1566		return;
1567
1568	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1569					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
 
 
 
1570		return;
1571
1572	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1573	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1574		LC_CURRENT_DATA_RATE_SHIFT;
1575	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1576		if (current_data_rate == 2) {
1577			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1578			return;
1579		}
1580		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1581	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1582		if (current_data_rate == 1) {
1583			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1584			return;
1585		}
1586		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1587	}
1588
1589	bridge_pos = pci_pcie_cap(root);
1590	if (!bridge_pos)
1591		return;
1592
1593	gpu_pos = pci_pcie_cap(adev->pdev);
1594	if (!gpu_pos)
1595		return;
1596
1597	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1598		if (current_data_rate != 2) {
1599			u16 bridge_cfg, gpu_cfg;
1600			u16 bridge_cfg2, gpu_cfg2;
1601			u32 max_lw, current_lw, tmp;
1602
1603			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1604			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1605
1606			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1607			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1608
1609			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1610			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1611
1612			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1613			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1614			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1615
1616			if (current_lw < max_lw) {
1617				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1618				if (tmp & LC_RENEGOTIATION_SUPPORT) {
1619					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1620					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1621					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1622					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1623				}
1624			}
1625
1626			for (i = 0; i < 10; i++) {
1627				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1628				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1629					break;
1630
1631				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1632				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1633
1634				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1635				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1636
1637				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1638				tmp |= LC_SET_QUIESCE;
1639				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1640
1641				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1642				tmp |= LC_REDO_EQ;
1643				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1644
1645				mdelay(100);
1646
1647				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1648				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1649				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1650				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1651
1652				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1653				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1654				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1655				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1656
1657				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1658				tmp16 &= ~((1 << 4) | (7 << 9));
1659				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1660				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1661
1662				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1663				tmp16 &= ~((1 << 4) | (7 << 9));
1664				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1665				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1666
1667				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1668				tmp &= ~LC_SET_QUIESCE;
1669				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1670			}
1671		}
1672	}
1673
1674	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1675	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1676	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1677
1678	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1679	tmp16 &= ~0xf;
1680	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1681		tmp16 |= 3;
1682	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1683		tmp16 |= 2;
1684	else
1685		tmp16 |= 1;
1686	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1687
1688	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1689	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1690	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1691
1692	for (i = 0; i < adev->usec_timeout; i++) {
1693		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1694		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1695			break;
1696		udelay(1);
1697	}
1698}
1699
1700static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1701{
1702	unsigned long flags;
1703	u32 r;
1704
1705	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1706	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1707	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1708	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1709	return r;
1710}
1711
1712static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1713{
1714	unsigned long flags;
1715
1716	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1717	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1718	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1719	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1720}
1721
1722static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1723{
1724	unsigned long flags;
1725	u32 r;
1726
1727	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1728	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1729	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1730	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1731	return r;
1732}
1733
1734static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1735{
1736	unsigned long flags;
1737
1738	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1739	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1740	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1741	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1742}
1743static void si_program_aspm(struct amdgpu_device *adev)
1744{
1745	u32 data, orig;
1746	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1747	bool disable_clkreq = false;
1748
1749	if (amdgpu_aspm == 0)
1750		return;
1751
1752	if (adev->flags & AMD_IS_APU)
1753		return;
1754	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1755	data &= ~LC_XMIT_N_FTS_MASK;
1756	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1757	if (orig != data)
1758		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1759
1760	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1761	data |= LC_GO_TO_RECOVERY;
1762	if (orig != data)
1763		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1764
1765	orig = data = RREG32_PCIE(PCIE_P_CNTL);
1766	data |= P_IGNORE_EDB_ERR;
1767	if (orig != data)
1768		WREG32_PCIE(PCIE_P_CNTL, data);
1769
1770	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1771	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1772	data |= LC_PMI_TO_L1_DIS;
1773	if (!disable_l0s)
1774		data |= LC_L0S_INACTIVITY(7);
1775
1776	if (!disable_l1) {
1777		data |= LC_L1_INACTIVITY(7);
1778		data &= ~LC_PMI_TO_L1_DIS;
1779		if (orig != data)
1780			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1781
1782		if (!disable_plloff_in_l1) {
1783			bool clk_req_support;
1784
1785			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1786			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1787			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1788			if (orig != data)
1789				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1790
1791			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1792			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1793			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1794			if (orig != data)
1795				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1796
1797			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1798			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1799			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1800			if (orig != data)
1801				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1802
1803			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1804			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1805			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1806			if (orig != data)
1807				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1808
1809			if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
1810				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1811				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1812				if (orig != data)
1813					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1814
1815				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1816				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1817				if (orig != data)
1818					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1819
1820				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1821				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1822				if (orig != data)
1823					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1824
1825				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1826				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1827				if (orig != data)
1828					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1829
1830				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1831				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1832				if (orig != data)
1833					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1834
1835				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1836				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1837				if (orig != data)
1838					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1839
1840				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1841				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1842				if (orig != data)
1843					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1844
1845				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1846				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1847				if (orig != data)
1848					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1849			}
1850			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1851			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1852			data |= LC_DYN_LANES_PWR_STATE(3);
1853			if (orig != data)
1854				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1855
1856			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1857			data &= ~LS2_EXIT_TIME_MASK;
1858			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1859				data |= LS2_EXIT_TIME(5);
1860			if (orig != data)
1861				si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1862
1863			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1864			data &= ~LS2_EXIT_TIME_MASK;
1865			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1866				data |= LS2_EXIT_TIME(5);
1867			if (orig != data)
1868				si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1869
1870			if (!disable_clkreq &&
1871			    !pci_is_root_bus(adev->pdev->bus)) {
1872				struct pci_dev *root = adev->pdev->bus->self;
1873				u32 lnkcap;
1874
1875				clk_req_support = false;
1876				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1877				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1878					clk_req_support = true;
1879			} else {
1880				clk_req_support = false;
1881			}
1882
1883			if (clk_req_support) {
1884				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
1885				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1886				if (orig != data)
1887					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
1888
1889				orig = data = RREG32(THM_CLK_CNTL);
1890				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1891				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1892				if (orig != data)
1893					WREG32(THM_CLK_CNTL, data);
1894
1895				orig = data = RREG32(MISC_CLK_CNTL);
1896				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1897				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1898				if (orig != data)
1899					WREG32(MISC_CLK_CNTL, data);
1900
1901				orig = data = RREG32(CG_CLKPIN_CNTL);
1902				data &= ~BCLK_AS_XCLK;
1903				if (orig != data)
1904					WREG32(CG_CLKPIN_CNTL, data);
1905
1906				orig = data = RREG32(CG_CLKPIN_CNTL_2);
1907				data &= ~FORCE_BIF_REFCLK_EN;
1908				if (orig != data)
1909					WREG32(CG_CLKPIN_CNTL_2, data);
1910
1911				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1912				data &= ~MPLL_CLKOUT_SEL_MASK;
1913				data |= MPLL_CLKOUT_SEL(4);
1914				if (orig != data)
1915					WREG32(MPLL_BYPASSCLK_SEL, data);
1916
1917				orig = data = RREG32(SPLL_CNTL_MODE);
1918				data &= ~SPLL_REFCLK_SEL_MASK;
1919				if (orig != data)
1920					WREG32(SPLL_CNTL_MODE, data);
1921			}
1922		}
1923	} else {
1924		if (orig != data)
1925			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1926	}
1927
1928	orig = data = RREG32_PCIE(PCIE_CNTL2);
1929	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
1930	if (orig != data)
1931		WREG32_PCIE(PCIE_CNTL2, data);
1932
1933	if (!disable_l0s) {
1934		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1935		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
1936			data = RREG32_PCIE(PCIE_LC_STATUS1);
1937			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
1938				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1939				data &= ~LC_L0S_INACTIVITY_MASK;
1940				if (orig != data)
1941					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1942			}
1943		}
1944	}
1945}
1946
1947static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
1948{
1949	int readrq;
1950	u16 v;
1951
1952	readrq = pcie_get_readrq(adev->pdev);
1953	v = ffs(readrq) - 8;
1954	if ((v == 0) || (v == 6) || (v == 7))
1955		pcie_set_readrq(adev->pdev, 512);
1956}
1957
1958static int si_common_hw_init(void *handle)
1959{
1960	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1961
1962	si_fix_pci_max_read_req_size(adev);
1963	si_init_golden_registers(adev);
1964	si_pcie_gen3_enable(adev);
1965	si_program_aspm(adev);
1966
1967	return 0;
1968}
1969
1970static int si_common_hw_fini(void *handle)
1971{
1972	return 0;
1973}
1974
1975static int si_common_suspend(void *handle)
1976{
1977	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1978
1979	return si_common_hw_fini(adev);
1980}
1981
1982static int si_common_resume(void *handle)
1983{
1984	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1985
1986	return si_common_hw_init(adev);
1987}
1988
1989static bool si_common_is_idle(void *handle)
1990{
1991	return true;
1992}
1993
1994static int si_common_wait_for_idle(void *handle)
1995{
1996	return 0;
1997}
1998
1999static int si_common_soft_reset(void *handle)
2000{
2001	return 0;
2002}
2003
2004static int si_common_set_clockgating_state(void *handle,
2005					    enum amd_clockgating_state state)
2006{
2007	return 0;
2008}
2009
2010static int si_common_set_powergating_state(void *handle,
2011					    enum amd_powergating_state state)
2012{
2013	return 0;
2014}
2015
2016static const struct amd_ip_funcs si_common_ip_funcs = {
2017	.name = "si_common",
2018	.early_init = si_common_early_init,
2019	.late_init = NULL,
2020	.sw_init = si_common_sw_init,
2021	.sw_fini = si_common_sw_fini,
2022	.hw_init = si_common_hw_init,
2023	.hw_fini = si_common_hw_fini,
2024	.suspend = si_common_suspend,
2025	.resume = si_common_resume,
2026	.is_idle = si_common_is_idle,
2027	.wait_for_idle = si_common_wait_for_idle,
2028	.soft_reset = si_common_soft_reset,
2029	.set_clockgating_state = si_common_set_clockgating_state,
2030	.set_powergating_state = si_common_set_powergating_state,
2031};
2032
2033static const struct amdgpu_ip_block_version si_common_ip_block =
2034{
2035	.type = AMD_IP_BLOCK_TYPE_COMMON,
2036	.major = 1,
2037	.minor = 0,
2038	.rev = 0,
2039	.funcs = &si_common_ip_funcs,
2040};
2041
2042int si_set_ip_blocks(struct amdgpu_device *adev)
2043{
2044	si_detect_hw_virtualization(adev);
2045
2046	switch (adev->asic_type) {
2047	case CHIP_VERDE:
2048	case CHIP_TAHITI:
2049	case CHIP_PITCAIRN:
2050		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2051		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2052		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2053		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2054		if (adev->enable_virtual_display)
2055			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2056		else
2057			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2058		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2059		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2060		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
2061		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2062		break;
2063	case CHIP_OLAND:
2064		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2065		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2066		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2067		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2068		if (adev->enable_virtual_display)
2069			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2070		else
2071			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2072		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2073		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2074		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
2075		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
2076		break;
2077	case CHIP_HAINAN:
2078		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2079		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2080		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2081		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2082		if (adev->enable_virtual_display)
2083			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2084		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2085		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2086		break;
2087	default:
2088		BUG();
2089	}
2090	return 0;
2091}
2092