1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/module.h>
  26
  27#include "amdgpu.h"
  28#include "amdgpu_ih.h"
  29#include "amdgpu_gfx.h"
  30#include "cikd.h"
  31#include "cik.h"
  32#include "cik_structs.h"
  33#include "atom.h"
  34#include "amdgpu_ucode.h"
  35#include "clearstate_ci.h"
  36
  37#include "dce/dce_8_0_d.h"
  38#include "dce/dce_8_0_sh_mask.h"
  39
  40#include "bif/bif_4_1_d.h"
  41#include "bif/bif_4_1_sh_mask.h"
  42
  43#include "gca/gfx_7_0_d.h"
  44#include "gca/gfx_7_2_enum.h"
  45#include "gca/gfx_7_2_sh_mask.h"
  46
  47#include "gmc/gmc_7_0_d.h"
  48#include "gmc/gmc_7_0_sh_mask.h"
  49
  50#include "oss/oss_2_0_d.h"
  51#include "oss/oss_2_0_sh_mask.h"
  52
  53#define NUM_SIMD_PER_CU 0x4 /* missing from the gfx_7 IP headers */
  54
  55#define GFX7_NUM_GFX_RINGS     1
  56#define GFX7_MEC_HPD_SIZE      2048
  57
  58static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev);
  59static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev);
  60static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev);
  61
  62MODULE_FIRMWARE("amdgpu/bonaire_pfp.bin");
  63MODULE_FIRMWARE("amdgpu/bonaire_me.bin");
  64MODULE_FIRMWARE("amdgpu/bonaire_ce.bin");
  65MODULE_FIRMWARE("amdgpu/bonaire_rlc.bin");
  66MODULE_FIRMWARE("amdgpu/bonaire_mec.bin");
  67
  68MODULE_FIRMWARE("amdgpu/hawaii_pfp.bin");
  69MODULE_FIRMWARE("amdgpu/hawaii_me.bin");
  70MODULE_FIRMWARE("amdgpu/hawaii_ce.bin");
  71MODULE_FIRMWARE("amdgpu/hawaii_rlc.bin");
  72MODULE_FIRMWARE("amdgpu/hawaii_mec.bin");
  73
  74MODULE_FIRMWARE("amdgpu/kaveri_pfp.bin");
  75MODULE_FIRMWARE("amdgpu/kaveri_me.bin");
  76MODULE_FIRMWARE("amdgpu/kaveri_ce.bin");
  77MODULE_FIRMWARE("amdgpu/kaveri_rlc.bin");
  78MODULE_FIRMWARE("amdgpu/kaveri_mec.bin");
  79MODULE_FIRMWARE("amdgpu/kaveri_mec2.bin");
  80
  81MODULE_FIRMWARE("amdgpu/kabini_pfp.bin");
  82MODULE_FIRMWARE("amdgpu/kabini_me.bin");
  83MODULE_FIRMWARE("amdgpu/kabini_ce.bin");
  84MODULE_FIRMWARE("amdgpu/kabini_rlc.bin");
  85MODULE_FIRMWARE("amdgpu/kabini_mec.bin");
  86
  87MODULE_FIRMWARE("amdgpu/mullins_pfp.bin");
  88MODULE_FIRMWARE("amdgpu/mullins_me.bin");
  89MODULE_FIRMWARE("amdgpu/mullins_ce.bin");
  90MODULE_FIRMWARE("amdgpu/mullins_rlc.bin");
  91MODULE_FIRMWARE("amdgpu/mullins_mec.bin");
  92
  93static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = {
 
  94	{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
  95	{mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
  96	{mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
  97	{mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
  98	{mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
  99	{mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
 100	{mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
 101	{mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
 102	{mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
 103	{mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
 104	{mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
 105	{mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
 106	{mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
 107	{mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
 108	{mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
 109	{mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
 110};
 111
 112static const u32 spectre_rlc_save_restore_register_list[] = {
 
 113	(0x0e00 << 16) | (0xc12c >> 2),
 114	0x00000000,
 115	(0x0e00 << 16) | (0xc140 >> 2),
 116	0x00000000,
 117	(0x0e00 << 16) | (0xc150 >> 2),
 118	0x00000000,
 119	(0x0e00 << 16) | (0xc15c >> 2),
 120	0x00000000,
 121	(0x0e00 << 16) | (0xc168 >> 2),
 122	0x00000000,
 123	(0x0e00 << 16) | (0xc170 >> 2),
 124	0x00000000,
 125	(0x0e00 << 16) | (0xc178 >> 2),
 126	0x00000000,
 127	(0x0e00 << 16) | (0xc204 >> 2),
 128	0x00000000,
 129	(0x0e00 << 16) | (0xc2b4 >> 2),
 130	0x00000000,
 131	(0x0e00 << 16) | (0xc2b8 >> 2),
 132	0x00000000,
 133	(0x0e00 << 16) | (0xc2bc >> 2),
 134	0x00000000,
 135	(0x0e00 << 16) | (0xc2c0 >> 2),
 136	0x00000000,
 137	(0x0e00 << 16) | (0x8228 >> 2),
 138	0x00000000,
 139	(0x0e00 << 16) | (0x829c >> 2),
 140	0x00000000,
 141	(0x0e00 << 16) | (0x869c >> 2),
 142	0x00000000,
 143	(0x0600 << 16) | (0x98f4 >> 2),
 144	0x00000000,
 145	(0x0e00 << 16) | (0x98f8 >> 2),
 146	0x00000000,
 147	(0x0e00 << 16) | (0x9900 >> 2),
 148	0x00000000,
 149	(0x0e00 << 16) | (0xc260 >> 2),
 150	0x00000000,
 151	(0x0e00 << 16) | (0x90e8 >> 2),
 152	0x00000000,
 153	(0x0e00 << 16) | (0x3c000 >> 2),
 154	0x00000000,
 155	(0x0e00 << 16) | (0x3c00c >> 2),
 156	0x00000000,
 157	(0x0e00 << 16) | (0x8c1c >> 2),
 158	0x00000000,
 159	(0x0e00 << 16) | (0x9700 >> 2),
 160	0x00000000,
 161	(0x0e00 << 16) | (0xcd20 >> 2),
 162	0x00000000,
 163	(0x4e00 << 16) | (0xcd20 >> 2),
 164	0x00000000,
 165	(0x5e00 << 16) | (0xcd20 >> 2),
 166	0x00000000,
 167	(0x6e00 << 16) | (0xcd20 >> 2),
 168	0x00000000,
 169	(0x7e00 << 16) | (0xcd20 >> 2),
 170	0x00000000,
 171	(0x8e00 << 16) | (0xcd20 >> 2),
 172	0x00000000,
 173	(0x9e00 << 16) | (0xcd20 >> 2),
 174	0x00000000,
 175	(0xae00 << 16) | (0xcd20 >> 2),
 176	0x00000000,
 177	(0xbe00 << 16) | (0xcd20 >> 2),
 178	0x00000000,
 179	(0x0e00 << 16) | (0x89bc >> 2),
 180	0x00000000,
 181	(0x0e00 << 16) | (0x8900 >> 2),
 182	0x00000000,
 183	0x3,
 184	(0x0e00 << 16) | (0xc130 >> 2),
 185	0x00000000,
 186	(0x0e00 << 16) | (0xc134 >> 2),
 187	0x00000000,
 188	(0x0e00 << 16) | (0xc1fc >> 2),
 189	0x00000000,
 190	(0x0e00 << 16) | (0xc208 >> 2),
 191	0x00000000,
 192	(0x0e00 << 16) | (0xc264 >> 2),
 193	0x00000000,
 194	(0x0e00 << 16) | (0xc268 >> 2),
 195	0x00000000,
 196	(0x0e00 << 16) | (0xc26c >> 2),
 197	0x00000000,
 198	(0x0e00 << 16) | (0xc270 >> 2),
 199	0x00000000,
 200	(0x0e00 << 16) | (0xc274 >> 2),
 201	0x00000000,
 202	(0x0e00 << 16) | (0xc278 >> 2),
 203	0x00000000,
 204	(0x0e00 << 16) | (0xc27c >> 2),
 205	0x00000000,
 206	(0x0e00 << 16) | (0xc280 >> 2),
 207	0x00000000,
 208	(0x0e00 << 16) | (0xc284 >> 2),
 209	0x00000000,
 210	(0x0e00 << 16) | (0xc288 >> 2),
 211	0x00000000,
 212	(0x0e00 << 16) | (0xc28c >> 2),
 213	0x00000000,
 214	(0x0e00 << 16) | (0xc290 >> 2),
 215	0x00000000,
 216	(0x0e00 << 16) | (0xc294 >> 2),
 217	0x00000000,
 218	(0x0e00 << 16) | (0xc298 >> 2),
 219	0x00000000,
 220	(0x0e00 << 16) | (0xc29c >> 2),
 221	0x00000000,
 222	(0x0e00 << 16) | (0xc2a0 >> 2),
 223	0x00000000,
 224	(0x0e00 << 16) | (0xc2a4 >> 2),
 225	0x00000000,
 226	(0x0e00 << 16) | (0xc2a8 >> 2),
 227	0x00000000,
 228	(0x0e00 << 16) | (0xc2ac  >> 2),
 229	0x00000000,
 230	(0x0e00 << 16) | (0xc2b0 >> 2),
 231	0x00000000,
 232	(0x0e00 << 16) | (0x301d0 >> 2),
 233	0x00000000,
 234	(0x0e00 << 16) | (0x30238 >> 2),
 235	0x00000000,
 236	(0x0e00 << 16) | (0x30250 >> 2),
 237	0x00000000,
 238	(0x0e00 << 16) | (0x30254 >> 2),
 239	0x00000000,
 240	(0x0e00 << 16) | (0x30258 >> 2),
 241	0x00000000,
 242	(0x0e00 << 16) | (0x3025c >> 2),
 243	0x00000000,
 244	(0x4e00 << 16) | (0xc900 >> 2),
 245	0x00000000,
 246	(0x5e00 << 16) | (0xc900 >> 2),
 247	0x00000000,
 248	(0x6e00 << 16) | (0xc900 >> 2),
 249	0x00000000,
 250	(0x7e00 << 16) | (0xc900 >> 2),
 251	0x00000000,
 252	(0x8e00 << 16) | (0xc900 >> 2),
 253	0x00000000,
 254	(0x9e00 << 16) | (0xc900 >> 2),
 255	0x00000000,
 256	(0xae00 << 16) | (0xc900 >> 2),
 257	0x00000000,
 258	(0xbe00 << 16) | (0xc900 >> 2),
 259	0x00000000,
 260	(0x4e00 << 16) | (0xc904 >> 2),
 261	0x00000000,
 262	(0x5e00 << 16) | (0xc904 >> 2),
 263	0x00000000,
 264	(0x6e00 << 16) | (0xc904 >> 2),
 265	0x00000000,
 266	(0x7e00 << 16) | (0xc904 >> 2),
 267	0x00000000,
 268	(0x8e00 << 16) | (0xc904 >> 2),
 269	0x00000000,
 270	(0x9e00 << 16) | (0xc904 >> 2),
 271	0x00000000,
 272	(0xae00 << 16) | (0xc904 >> 2),
 273	0x00000000,
 274	(0xbe00 << 16) | (0xc904 >> 2),
 275	0x00000000,
 276	(0x4e00 << 16) | (0xc908 >> 2),
 277	0x00000000,
 278	(0x5e00 << 16) | (0xc908 >> 2),
 279	0x00000000,
 280	(0x6e00 << 16) | (0xc908 >> 2),
 281	0x00000000,
 282	(0x7e00 << 16) | (0xc908 >> 2),
 283	0x00000000,
 284	(0x8e00 << 16) | (0xc908 >> 2),
 285	0x00000000,
 286	(0x9e00 << 16) | (0xc908 >> 2),
 287	0x00000000,
 288	(0xae00 << 16) | (0xc908 >> 2),
 289	0x00000000,
 290	(0xbe00 << 16) | (0xc908 >> 2),
 291	0x00000000,
 292	(0x4e00 << 16) | (0xc90c >> 2),
 293	0x00000000,
 294	(0x5e00 << 16) | (0xc90c >> 2),
 295	0x00000000,
 296	(0x6e00 << 16) | (0xc90c >> 2),
 297	0x00000000,
 298	(0x7e00 << 16) | (0xc90c >> 2),
 299	0x00000000,
 300	(0x8e00 << 16) | (0xc90c >> 2),
 301	0x00000000,
 302	(0x9e00 << 16) | (0xc90c >> 2),
 303	0x00000000,
 304	(0xae00 << 16) | (0xc90c >> 2),
 305	0x00000000,
 306	(0xbe00 << 16) | (0xc90c >> 2),
 307	0x00000000,
 308	(0x4e00 << 16) | (0xc910 >> 2),
 309	0x00000000,
 310	(0x5e00 << 16) | (0xc910 >> 2),
 311	0x00000000,
 312	(0x6e00 << 16) | (0xc910 >> 2),
 313	0x00000000,
 314	(0x7e00 << 16) | (0xc910 >> 2),
 315	0x00000000,
 316	(0x8e00 << 16) | (0xc910 >> 2),
 317	0x00000000,
 318	(0x9e00 << 16) | (0xc910 >> 2),
 319	0x00000000,
 320	(0xae00 << 16) | (0xc910 >> 2),
 321	0x00000000,
 322	(0xbe00 << 16) | (0xc910 >> 2),
 323	0x00000000,
 324	(0x0e00 << 16) | (0xc99c >> 2),
 325	0x00000000,
 326	(0x0e00 << 16) | (0x9834 >> 2),
 327	0x00000000,
 328	(0x0000 << 16) | (0x30f00 >> 2),
 329	0x00000000,
 330	(0x0001 << 16) | (0x30f00 >> 2),
 331	0x00000000,
 332	(0x0000 << 16) | (0x30f04 >> 2),
 333	0x00000000,
 334	(0x0001 << 16) | (0x30f04 >> 2),
 335	0x00000000,
 336	(0x0000 << 16) | (0x30f08 >> 2),
 337	0x00000000,
 338	(0x0001 << 16) | (0x30f08 >> 2),
 339	0x00000000,
 340	(0x0000 << 16) | (0x30f0c >> 2),
 341	0x00000000,
 342	(0x0001 << 16) | (0x30f0c >> 2),
 343	0x00000000,
 344	(0x0600 << 16) | (0x9b7c >> 2),
 345	0x00000000,
 346	(0x0e00 << 16) | (0x8a14 >> 2),
 347	0x00000000,
 348	(0x0e00 << 16) | (0x8a18 >> 2),
 349	0x00000000,
 350	(0x0600 << 16) | (0x30a00 >> 2),
 351	0x00000000,
 352	(0x0e00 << 16) | (0x8bf0 >> 2),
 353	0x00000000,
 354	(0x0e00 << 16) | (0x8bcc >> 2),
 355	0x00000000,
 356	(0x0e00 << 16) | (0x8b24 >> 2),
 357	0x00000000,
 358	(0x0e00 << 16) | (0x30a04 >> 2),
 359	0x00000000,
 360	(0x0600 << 16) | (0x30a10 >> 2),
 361	0x00000000,
 362	(0x0600 << 16) | (0x30a14 >> 2),
 363	0x00000000,
 364	(0x0600 << 16) | (0x30a18 >> 2),
 365	0x00000000,
 366	(0x0600 << 16) | (0x30a2c >> 2),
 367	0x00000000,
 368	(0x0e00 << 16) | (0xc700 >> 2),
 369	0x00000000,
 370	(0x0e00 << 16) | (0xc704 >> 2),
 371	0x00000000,
 372	(0x0e00 << 16) | (0xc708 >> 2),
 373	0x00000000,
 374	(0x0e00 << 16) | (0xc768 >> 2),
 375	0x00000000,
 376	(0x0400 << 16) | (0xc770 >> 2),
 377	0x00000000,
 378	(0x0400 << 16) | (0xc774 >> 2),
 379	0x00000000,
 380	(0x0400 << 16) | (0xc778 >> 2),
 381	0x00000000,
 382	(0x0400 << 16) | (0xc77c >> 2),
 383	0x00000000,
 384	(0x0400 << 16) | (0xc780 >> 2),
 385	0x00000000,
 386	(0x0400 << 16) | (0xc784 >> 2),
 387	0x00000000,
 388	(0x0400 << 16) | (0xc788 >> 2),
 389	0x00000000,
 390	(0x0400 << 16) | (0xc78c >> 2),
 391	0x00000000,
 392	(0x0400 << 16) | (0xc798 >> 2),
 393	0x00000000,
 394	(0x0400 << 16) | (0xc79c >> 2),
 395	0x00000000,
 396	(0x0400 << 16) | (0xc7a0 >> 2),
 397	0x00000000,
 398	(0x0400 << 16) | (0xc7a4 >> 2),
 399	0x00000000,
 400	(0x0400 << 16) | (0xc7a8 >> 2),
 401	0x00000000,
 402	(0x0400 << 16) | (0xc7ac >> 2),
 403	0x00000000,
 404	(0x0400 << 16) | (0xc7b0 >> 2),
 405	0x00000000,
 406	(0x0400 << 16) | (0xc7b4 >> 2),
 407	0x00000000,
 408	(0x0e00 << 16) | (0x9100 >> 2),
 409	0x00000000,
 410	(0x0e00 << 16) | (0x3c010 >> 2),
 411	0x00000000,
 412	(0x0e00 << 16) | (0x92a8 >> 2),
 413	0x00000000,
 414	(0x0e00 << 16) | (0x92ac >> 2),
 415	0x00000000,
 416	(0x0e00 << 16) | (0x92b4 >> 2),
 417	0x00000000,
 418	(0x0e00 << 16) | (0x92b8 >> 2),
 419	0x00000000,
 420	(0x0e00 << 16) | (0x92bc >> 2),
 421	0x00000000,
 422	(0x0e00 << 16) | (0x92c0 >> 2),
 423	0x00000000,
 424	(0x0e00 << 16) | (0x92c4 >> 2),
 425	0x00000000,
 426	(0x0e00 << 16) | (0x92c8 >> 2),
 427	0x00000000,
 428	(0x0e00 << 16) | (0x92cc >> 2),
 429	0x00000000,
 430	(0x0e00 << 16) | (0x92d0 >> 2),
 431	0x00000000,
 432	(0x0e00 << 16) | (0x8c00 >> 2),
 433	0x00000000,
 434	(0x0e00 << 16) | (0x8c04 >> 2),
 435	0x00000000,
 436	(0x0e00 << 16) | (0x8c20 >> 2),
 437	0x00000000,
 438	(0x0e00 << 16) | (0x8c38 >> 2),
 439	0x00000000,
 440	(0x0e00 << 16) | (0x8c3c >> 2),
 441	0x00000000,
 442	(0x0e00 << 16) | (0xae00 >> 2),
 443	0x00000000,
 444	(0x0e00 << 16) | (0x9604 >> 2),
 445	0x00000000,
 446	(0x0e00 << 16) | (0xac08 >> 2),
 447	0x00000000,
 448	(0x0e00 << 16) | (0xac0c >> 2),
 449	0x00000000,
 450	(0x0e00 << 16) | (0xac10 >> 2),
 451	0x00000000,
 452	(0x0e00 << 16) | (0xac14 >> 2),
 453	0x00000000,
 454	(0x0e00 << 16) | (0xac58 >> 2),
 455	0x00000000,
 456	(0x0e00 << 16) | (0xac68 >> 2),
 457	0x00000000,
 458	(0x0e00 << 16) | (0xac6c >> 2),
 459	0x00000000,
 460	(0x0e00 << 16) | (0xac70 >> 2),
 461	0x00000000,
 462	(0x0e00 << 16) | (0xac74 >> 2),
 463	0x00000000,
 464	(0x0e00 << 16) | (0xac78 >> 2),
 465	0x00000000,
 466	(0x0e00 << 16) | (0xac7c >> 2),
 467	0x00000000,
 468	(0x0e00 << 16) | (0xac80 >> 2),
 469	0x00000000,
 470	(0x0e00 << 16) | (0xac84 >> 2),
 471	0x00000000,
 472	(0x0e00 << 16) | (0xac88 >> 2),
 473	0x00000000,
 474	(0x0e00 << 16) | (0xac8c >> 2),
 475	0x00000000,
 476	(0x0e00 << 16) | (0x970c >> 2),
 477	0x00000000,
 478	(0x0e00 << 16) | (0x9714 >> 2),
 479	0x00000000,
 480	(0x0e00 << 16) | (0x9718 >> 2),
 481	0x00000000,
 482	(0x0e00 << 16) | (0x971c >> 2),
 483	0x00000000,
 484	(0x0e00 << 16) | (0x31068 >> 2),
 485	0x00000000,
 486	(0x4e00 << 16) | (0x31068 >> 2),
 487	0x00000000,
 488	(0x5e00 << 16) | (0x31068 >> 2),
 489	0x00000000,
 490	(0x6e00 << 16) | (0x31068 >> 2),
 491	0x00000000,
 492	(0x7e00 << 16) | (0x31068 >> 2),
 493	0x00000000,
 494	(0x8e00 << 16) | (0x31068 >> 2),
 495	0x00000000,
 496	(0x9e00 << 16) | (0x31068 >> 2),
 497	0x00000000,
 498	(0xae00 << 16) | (0x31068 >> 2),
 499	0x00000000,
 500	(0xbe00 << 16) | (0x31068 >> 2),
 501	0x00000000,
 502	(0x0e00 << 16) | (0xcd10 >> 2),
 503	0x00000000,
 504	(0x0e00 << 16) | (0xcd14 >> 2),
 505	0x00000000,
 506	(0x0e00 << 16) | (0x88b0 >> 2),
 507	0x00000000,
 508	(0x0e00 << 16) | (0x88b4 >> 2),
 509	0x00000000,
 510	(0x0e00 << 16) | (0x88b8 >> 2),
 511	0x00000000,
 512	(0x0e00 << 16) | (0x88bc >> 2),
 513	0x00000000,
 514	(0x0400 << 16) | (0x89c0 >> 2),
 515	0x00000000,
 516	(0x0e00 << 16) | (0x88c4 >> 2),
 517	0x00000000,
 518	(0x0e00 << 16) | (0x88c8 >> 2),
 519	0x00000000,
 520	(0x0e00 << 16) | (0x88d0 >> 2),
 521	0x00000000,
 522	(0x0e00 << 16) | (0x88d4 >> 2),
 523	0x00000000,
 524	(0x0e00 << 16) | (0x88d8 >> 2),
 525	0x00000000,
 526	(0x0e00 << 16) | (0x8980 >> 2),
 527	0x00000000,
 528	(0x0e00 << 16) | (0x30938 >> 2),
 529	0x00000000,
 530	(0x0e00 << 16) | (0x3093c >> 2),
 531	0x00000000,
 532	(0x0e00 << 16) | (0x30940 >> 2),
 533	0x00000000,
 534	(0x0e00 << 16) | (0x89a0 >> 2),
 535	0x00000000,
 536	(0x0e00 << 16) | (0x30900 >> 2),
 537	0x00000000,
 538	(0x0e00 << 16) | (0x30904 >> 2),
 539	0x00000000,
 540	(0x0e00 << 16) | (0x89b4 >> 2),
 541	0x00000000,
 542	(0x0e00 << 16) | (0x3c210 >> 2),
 543	0x00000000,
 544	(0x0e00 << 16) | (0x3c214 >> 2),
 545	0x00000000,
 546	(0x0e00 << 16) | (0x3c218 >> 2),
 547	0x00000000,
 548	(0x0e00 << 16) | (0x8904 >> 2),
 549	0x00000000,
 550	0x5,
 551	(0x0e00 << 16) | (0x8c28 >> 2),
 552	(0x0e00 << 16) | (0x8c2c >> 2),
 553	(0x0e00 << 16) | (0x8c30 >> 2),
 554	(0x0e00 << 16) | (0x8c34 >> 2),
 555	(0x0e00 << 16) | (0x9600 >> 2),
 556};
 557
 558static const u32 kalindi_rlc_save_restore_register_list[] = {
 
 559	(0x0e00 << 16) | (0xc12c >> 2),
 560	0x00000000,
 561	(0x0e00 << 16) | (0xc140 >> 2),
 562	0x00000000,
 563	(0x0e00 << 16) | (0xc150 >> 2),
 564	0x00000000,
 565	(0x0e00 << 16) | (0xc15c >> 2),
 566	0x00000000,
 567	(0x0e00 << 16) | (0xc168 >> 2),
 568	0x00000000,
 569	(0x0e00 << 16) | (0xc170 >> 2),
 570	0x00000000,
 571	(0x0e00 << 16) | (0xc204 >> 2),
 572	0x00000000,
 573	(0x0e00 << 16) | (0xc2b4 >> 2),
 574	0x00000000,
 575	(0x0e00 << 16) | (0xc2b8 >> 2),
 576	0x00000000,
 577	(0x0e00 << 16) | (0xc2bc >> 2),
 578	0x00000000,
 579	(0x0e00 << 16) | (0xc2c0 >> 2),
 580	0x00000000,
 581	(0x0e00 << 16) | (0x8228 >> 2),
 582	0x00000000,
 583	(0x0e00 << 16) | (0x829c >> 2),
 584	0x00000000,
 585	(0x0e00 << 16) | (0x869c >> 2),
 586	0x00000000,
 587	(0x0600 << 16) | (0x98f4 >> 2),
 588	0x00000000,
 589	(0x0e00 << 16) | (0x98f8 >> 2),
 590	0x00000000,
 591	(0x0e00 << 16) | (0x9900 >> 2),
 592	0x00000000,
 593	(0x0e00 << 16) | (0xc260 >> 2),
 594	0x00000000,
 595	(0x0e00 << 16) | (0x90e8 >> 2),
 596	0x00000000,
 597	(0x0e00 << 16) | (0x3c000 >> 2),
 598	0x00000000,
 599	(0x0e00 << 16) | (0x3c00c >> 2),
 600	0x00000000,
 601	(0x0e00 << 16) | (0x8c1c >> 2),
 602	0x00000000,
 603	(0x0e00 << 16) | (0x9700 >> 2),
 604	0x00000000,
 605	(0x0e00 << 16) | (0xcd20 >> 2),
 606	0x00000000,
 607	(0x4e00 << 16) | (0xcd20 >> 2),
 608	0x00000000,
 609	(0x5e00 << 16) | (0xcd20 >> 2),
 610	0x00000000,
 611	(0x6e00 << 16) | (0xcd20 >> 2),
 612	0x00000000,
 613	(0x7e00 << 16) | (0xcd20 >> 2),
 614	0x00000000,
 615	(0x0e00 << 16) | (0x89bc >> 2),
 616	0x00000000,
 617	(0x0e00 << 16) | (0x8900 >> 2),
 618	0x00000000,
 619	0x3,
 620	(0x0e00 << 16) | (0xc130 >> 2),
 621	0x00000000,
 622	(0x0e00 << 16) | (0xc134 >> 2),
 623	0x00000000,
 624	(0x0e00 << 16) | (0xc1fc >> 2),
 625	0x00000000,
 626	(0x0e00 << 16) | (0xc208 >> 2),
 627	0x00000000,
 628	(0x0e00 << 16) | (0xc264 >> 2),
 629	0x00000000,
 630	(0x0e00 << 16) | (0xc268 >> 2),
 631	0x00000000,
 632	(0x0e00 << 16) | (0xc26c >> 2),
 633	0x00000000,
 634	(0x0e00 << 16) | (0xc270 >> 2),
 635	0x00000000,
 636	(0x0e00 << 16) | (0xc274 >> 2),
 637	0x00000000,
 638	(0x0e00 << 16) | (0xc28c >> 2),
 639	0x00000000,
 640	(0x0e00 << 16) | (0xc290 >> 2),
 641	0x00000000,
 642	(0x0e00 << 16) | (0xc294 >> 2),
 643	0x00000000,
 644	(0x0e00 << 16) | (0xc298 >> 2),
 645	0x00000000,
 646	(0x0e00 << 16) | (0xc2a0 >> 2),
 647	0x00000000,
 648	(0x0e00 << 16) | (0xc2a4 >> 2),
 649	0x00000000,
 650	(0x0e00 << 16) | (0xc2a8 >> 2),
 651	0x00000000,
 652	(0x0e00 << 16) | (0xc2ac >> 2),
 653	0x00000000,
 654	(0x0e00 << 16) | (0x301d0 >> 2),
 655	0x00000000,
 656	(0x0e00 << 16) | (0x30238 >> 2),
 657	0x00000000,
 658	(0x0e00 << 16) | (0x30250 >> 2),
 659	0x00000000,
 660	(0x0e00 << 16) | (0x30254 >> 2),
 661	0x00000000,
 662	(0x0e00 << 16) | (0x30258 >> 2),
 663	0x00000000,
 664	(0x0e00 << 16) | (0x3025c >> 2),
 665	0x00000000,
 666	(0x4e00 << 16) | (0xc900 >> 2),
 667	0x00000000,
 668	(0x5e00 << 16) | (0xc900 >> 2),
 669	0x00000000,
 670	(0x6e00 << 16) | (0xc900 >> 2),
 671	0x00000000,
 672	(0x7e00 << 16) | (0xc900 >> 2),
 673	0x00000000,
 674	(0x4e00 << 16) | (0xc904 >> 2),
 675	0x00000000,
 676	(0x5e00 << 16) | (0xc904 >> 2),
 677	0x00000000,
 678	(0x6e00 << 16) | (0xc904 >> 2),
 679	0x00000000,
 680	(0x7e00 << 16) | (0xc904 >> 2),
 681	0x00000000,
 682	(0x4e00 << 16) | (0xc908 >> 2),
 683	0x00000000,
 684	(0x5e00 << 16) | (0xc908 >> 2),
 685	0x00000000,
 686	(0x6e00 << 16) | (0xc908 >> 2),
 687	0x00000000,
 688	(0x7e00 << 16) | (0xc908 >> 2),
 689	0x00000000,
 690	(0x4e00 << 16) | (0xc90c >> 2),
 691	0x00000000,
 692	(0x5e00 << 16) | (0xc90c >> 2),
 693	0x00000000,
 694	(0x6e00 << 16) | (0xc90c >> 2),
 695	0x00000000,
 696	(0x7e00 << 16) | (0xc90c >> 2),
 697	0x00000000,
 698	(0x4e00 << 16) | (0xc910 >> 2),
 699	0x00000000,
 700	(0x5e00 << 16) | (0xc910 >> 2),
 701	0x00000000,
 702	(0x6e00 << 16) | (0xc910 >> 2),
 703	0x00000000,
 704	(0x7e00 << 16) | (0xc910 >> 2),
 705	0x00000000,
 706	(0x0e00 << 16) | (0xc99c >> 2),
 707	0x00000000,
 708	(0x0e00 << 16) | (0x9834 >> 2),
 709	0x00000000,
 710	(0x0000 << 16) | (0x30f00 >> 2),
 711	0x00000000,
 712	(0x0000 << 16) | (0x30f04 >> 2),
 713	0x00000000,
 714	(0x0000 << 16) | (0x30f08 >> 2),
 715	0x00000000,
 716	(0x0000 << 16) | (0x30f0c >> 2),
 717	0x00000000,
 718	(0x0600 << 16) | (0x9b7c >> 2),
 719	0x00000000,
 720	(0x0e00 << 16) | (0x8a14 >> 2),
 721	0x00000000,
 722	(0x0e00 << 16) | (0x8a18 >> 2),
 723	0x00000000,
 724	(0x0600 << 16) | (0x30a00 >> 2),
 725	0x00000000,
 726	(0x0e00 << 16) | (0x8bf0 >> 2),
 727	0x00000000,
 728	(0x0e00 << 16) | (0x8bcc >> 2),
 729	0x00000000,
 730	(0x0e00 << 16) | (0x8b24 >> 2),
 731	0x00000000,
 732	(0x0e00 << 16) | (0x30a04 >> 2),
 733	0x00000000,
 734	(0x0600 << 16) | (0x30a10 >> 2),
 735	0x00000000,
 736	(0x0600 << 16) | (0x30a14 >> 2),
 737	0x00000000,
 738	(0x0600 << 16) | (0x30a18 >> 2),
 739	0x00000000,
 740	(0x0600 << 16) | (0x30a2c >> 2),
 741	0x00000000,
 742	(0x0e00 << 16) | (0xc700 >> 2),
 743	0x00000000,
 744	(0x0e00 << 16) | (0xc704 >> 2),
 745	0x00000000,
 746	(0x0e00 << 16) | (0xc708 >> 2),
 747	0x00000000,
 748	(0x0e00 << 16) | (0xc768 >> 2),
 749	0x00000000,
 750	(0x0400 << 16) | (0xc770 >> 2),
 751	0x00000000,
 752	(0x0400 << 16) | (0xc774 >> 2),
 753	0x00000000,
 754	(0x0400 << 16) | (0xc798 >> 2),
 755	0x00000000,
 756	(0x0400 << 16) | (0xc79c >> 2),
 757	0x00000000,
 758	(0x0e00 << 16) | (0x9100 >> 2),
 759	0x00000000,
 760	(0x0e00 << 16) | (0x3c010 >> 2),
 761	0x00000000,
 762	(0x0e00 << 16) | (0x8c00 >> 2),
 763	0x00000000,
 764	(0x0e00 << 16) | (0x8c04 >> 2),
 765	0x00000000,
 766	(0x0e00 << 16) | (0x8c20 >> 2),
 767	0x00000000,
 768	(0x0e00 << 16) | (0x8c38 >> 2),
 769	0x00000000,
 770	(0x0e00 << 16) | (0x8c3c >> 2),
 771	0x00000000,
 772	(0x0e00 << 16) | (0xae00 >> 2),
 773	0x00000000,
 774	(0x0e00 << 16) | (0x9604 >> 2),
 775	0x00000000,
 776	(0x0e00 << 16) | (0xac08 >> 2),
 777	0x00000000,
 778	(0x0e00 << 16) | (0xac0c >> 2),
 779	0x00000000,
 780	(0x0e00 << 16) | (0xac10 >> 2),
 781	0x00000000,
 782	(0x0e00 << 16) | (0xac14 >> 2),
 783	0x00000000,
 784	(0x0e00 << 16) | (0xac58 >> 2),
 785	0x00000000,
 786	(0x0e00 << 16) | (0xac68 >> 2),
 787	0x00000000,
 788	(0x0e00 << 16) | (0xac6c >> 2),
 789	0x00000000,
 790	(0x0e00 << 16) | (0xac70 >> 2),
 791	0x00000000,
 792	(0x0e00 << 16) | (0xac74 >> 2),
 793	0x00000000,
 794	(0x0e00 << 16) | (0xac78 >> 2),
 795	0x00000000,
 796	(0x0e00 << 16) | (0xac7c >> 2),
 797	0x00000000,
 798	(0x0e00 << 16) | (0xac80 >> 2),
 799	0x00000000,
 800	(0x0e00 << 16) | (0xac84 >> 2),
 801	0x00000000,
 802	(0x0e00 << 16) | (0xac88 >> 2),
 803	0x00000000,
 804	(0x0e00 << 16) | (0xac8c >> 2),
 805	0x00000000,
 806	(0x0e00 << 16) | (0x970c >> 2),
 807	0x00000000,
 808	(0x0e00 << 16) | (0x9714 >> 2),
 809	0x00000000,
 810	(0x0e00 << 16) | (0x9718 >> 2),
 811	0x00000000,
 812	(0x0e00 << 16) | (0x971c >> 2),
 813	0x00000000,
 814	(0x0e00 << 16) | (0x31068 >> 2),
 815	0x00000000,
 816	(0x4e00 << 16) | (0x31068 >> 2),
 817	0x00000000,
 818	(0x5e00 << 16) | (0x31068 >> 2),
 819	0x00000000,
 820	(0x6e00 << 16) | (0x31068 >> 2),
 821	0x00000000,
 822	(0x7e00 << 16) | (0x31068 >> 2),
 823	0x00000000,
 824	(0x0e00 << 16) | (0xcd10 >> 2),
 825	0x00000000,
 826	(0x0e00 << 16) | (0xcd14 >> 2),
 827	0x00000000,
 828	(0x0e00 << 16) | (0x88b0 >> 2),
 829	0x00000000,
 830	(0x0e00 << 16) | (0x88b4 >> 2),
 831	0x00000000,
 832	(0x0e00 << 16) | (0x88b8 >> 2),
 833	0x00000000,
 834	(0x0e00 << 16) | (0x88bc >> 2),
 835	0x00000000,
 836	(0x0400 << 16) | (0x89c0 >> 2),
 837	0x00000000,
 838	(0x0e00 << 16) | (0x88c4 >> 2),
 839	0x00000000,
 840	(0x0e00 << 16) | (0x88c8 >> 2),
 841	0x00000000,
 842	(0x0e00 << 16) | (0x88d0 >> 2),
 843	0x00000000,
 844	(0x0e00 << 16) | (0x88d4 >> 2),
 845	0x00000000,
 846	(0x0e00 << 16) | (0x88d8 >> 2),
 847	0x00000000,
 848	(0x0e00 << 16) | (0x8980 >> 2),
 849	0x00000000,
 850	(0x0e00 << 16) | (0x30938 >> 2),
 851	0x00000000,
 852	(0x0e00 << 16) | (0x3093c >> 2),
 853	0x00000000,
 854	(0x0e00 << 16) | (0x30940 >> 2),
 855	0x00000000,
 856	(0x0e00 << 16) | (0x89a0 >> 2),
 857	0x00000000,
 858	(0x0e00 << 16) | (0x30900 >> 2),
 859	0x00000000,
 860	(0x0e00 << 16) | (0x30904 >> 2),
 861	0x00000000,
 862	(0x0e00 << 16) | (0x89b4 >> 2),
 863	0x00000000,
 864	(0x0e00 << 16) | (0x3e1fc >> 2),
 865	0x00000000,
 866	(0x0e00 << 16) | (0x3c210 >> 2),
 867	0x00000000,
 868	(0x0e00 << 16) | (0x3c214 >> 2),
 869	0x00000000,
 870	(0x0e00 << 16) | (0x3c218 >> 2),
 871	0x00000000,
 872	(0x0e00 << 16) | (0x8904 >> 2),
 873	0x00000000,
 874	0x5,
 875	(0x0e00 << 16) | (0x8c28 >> 2),
 876	(0x0e00 << 16) | (0x8c2c >> 2),
 877	(0x0e00 << 16) | (0x8c30 >> 2),
 878	(0x0e00 << 16) | (0x8c34 >> 2),
 879	(0x0e00 << 16) | (0x9600 >> 2),
 880};
 881
 882static u32 gfx_v7_0_get_csb_size(struct amdgpu_device *adev);
 883static void gfx_v7_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer);
 884static void gfx_v7_0_init_pg(struct amdgpu_device *adev);
 885static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev);
 886
 887static void gfx_v7_0_free_microcode(struct amdgpu_device *adev)
 888{
 889	amdgpu_ucode_release(&adev->gfx.pfp_fw);
 890	amdgpu_ucode_release(&adev->gfx.me_fw);
 891	amdgpu_ucode_release(&adev->gfx.ce_fw);
 892	amdgpu_ucode_release(&adev->gfx.mec_fw);
 893	amdgpu_ucode_release(&adev->gfx.mec2_fw);
 894	amdgpu_ucode_release(&adev->gfx.rlc_fw);
 895}
 896
 897/*
 898 * Core functions
 899 */
 900/**
 901 * gfx_v7_0_init_microcode - load ucode images from disk
 902 *
 903 * @adev: amdgpu_device pointer
 904 *
 905 * Use the firmware interface to load the ucode images into
 906 * the driver (not loaded into hw).
 907 * Returns 0 on success, error on failure.
 908 */
 909static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 910{
 911	const char *chip_name;
 912	char fw_name[30];
 913	int err;
 914
 915	DRM_DEBUG("\n");
 916
 917	switch (adev->asic_type) {
 918	case CHIP_BONAIRE:
 919		chip_name = "bonaire";
 920		break;
 921	case CHIP_HAWAII:
 922		chip_name = "hawaii";
 923		break;
 924	case CHIP_KAVERI:
 925		chip_name = "kaveri";
 926		break;
 927	case CHIP_KABINI:
 928		chip_name = "kabini";
 929		break;
 930	case CHIP_MULLINS:
 931		chip_name = "mullins";
 932		break;
 933	default:
 934		BUG();
 935	}
 936
 937	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 938	err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name);
 
 
 
 939	if (err)
 940		goto out;
 941
 942	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 943	err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name);
 
 
 
 944	if (err)
 945		goto out;
 946
 947	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 948	err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name);
 
 
 
 949	if (err)
 950		goto out;
 951
 952	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
 953	err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);
 
 
 
 954	if (err)
 955		goto out;
 956
 957	if (adev->asic_type == CHIP_KAVERI) {
 958		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
 959		err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name);
 
 
 
 960		if (err)
 961			goto out;
 962	}
 963
 964	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 965	err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);
 966	if (err)
 967		goto out;
 
 
 968out:
 969	if (err) {
 970		pr_err("gfx7: Failed to load firmware \"%s\"\n", fw_name);
 971		gfx_v7_0_free_microcode(adev);
 
 
 
 
 
 
 
 
 
 
 
 972	}
 973	return err;
 974}
 975
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 976/**
 977 * gfx_v7_0_tiling_mode_table_init - init the hw tiling table
 978 *
 979 * @adev: amdgpu_device pointer
 980 *
 981 * Starting with SI, the tiling setup is done globally in a
 982 * set of 32 tiling modes.  Rather than selecting each set of
 983 * parameters per surface as on older asics, we just select
 984 * which index in the tiling table we want to use, and the
 985 * surface uses those parameters (CIK).
 986 */
 987static void gfx_v7_0_tiling_mode_table_init(struct amdgpu_device *adev)
 988{
 989	const u32 num_tile_mode_states =
 990			ARRAY_SIZE(adev->gfx.config.tile_mode_array);
 991	const u32 num_secondary_tile_mode_states =
 992			ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
 993	u32 reg_offset, split_equal_to_row_size;
 994	uint32_t *tile, *macrotile;
 995
 996	tile = adev->gfx.config.tile_mode_array;
 997	macrotile = adev->gfx.config.macrotile_mode_array;
 998
 999	switch (adev->gfx.config.mem_row_size_in_kb) {
1000	case 1:
1001		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1002		break;
1003	case 2:
1004	default:
1005		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1006		break;
1007	case 4:
1008		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1009		break;
1010	}
1011
1012	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1013		tile[reg_offset] = 0;
1014	for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1015		macrotile[reg_offset] = 0;
1016
1017	switch (adev->asic_type) {
1018	case CHIP_BONAIRE:
1019		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1020			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1021			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1022			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1023		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1024			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1025			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1026			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1027		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1028			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1029			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1030			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1031		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1032			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1033			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1034			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1035		tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1036			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1037			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1038			   TILE_SPLIT(split_equal_to_row_size));
1039		tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1040			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1041			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1042		tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1043			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1044			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1045			   TILE_SPLIT(split_equal_to_row_size));
1046		tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1047		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1048			   PIPE_CONFIG(ADDR_SURF_P4_16x16));
1049		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1050			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1051			   MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1052		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1053			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1054			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1055			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1056		tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1057			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1058			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1059			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1060		tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1061		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1062			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1063			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1064		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1065			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1066			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1067			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1068		tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1069			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1070			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1071			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1072		tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1073			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1074			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1075			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1076		tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1077		tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1078			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1079			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1080			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1081		tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1082			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1083			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1084		tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1085			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1086			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1087			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1088		tile[21] =  (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1089			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1090			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1091			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1092		tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1093			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1094			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1095			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1096		tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1097		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1098			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1099			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1100			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1101		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1102			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1103			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1104			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1105		tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1106			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1107			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1108			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1109		tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1110			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1111			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1112		tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1113			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1114			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1115			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1116		tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1117			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1118			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1119			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1120		tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1121
1122		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1123				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1124				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1125				NUM_BANKS(ADDR_SURF_16_BANK));
1126		macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1127				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1128				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1129				NUM_BANKS(ADDR_SURF_16_BANK));
1130		macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1131				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1132				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1133				NUM_BANKS(ADDR_SURF_16_BANK));
1134		macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1135				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1136				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1137				NUM_BANKS(ADDR_SURF_16_BANK));
1138		macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1139				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1140				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1141				NUM_BANKS(ADDR_SURF_16_BANK));
1142		macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1143				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1144				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1145				NUM_BANKS(ADDR_SURF_8_BANK));
1146		macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1147				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1148				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1149				NUM_BANKS(ADDR_SURF_4_BANK));
1150		macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1151				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1152				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1153				NUM_BANKS(ADDR_SURF_16_BANK));
1154		macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1155				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1156				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1157				NUM_BANKS(ADDR_SURF_16_BANK));
1158		macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1159				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1160				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1161				NUM_BANKS(ADDR_SURF_16_BANK));
1162		macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1163				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1164				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1165				NUM_BANKS(ADDR_SURF_16_BANK));
1166		macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1167				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1168				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1169				NUM_BANKS(ADDR_SURF_16_BANK));
1170		macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1171				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1172				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1173				NUM_BANKS(ADDR_SURF_8_BANK));
1174		macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1175				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1176				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1177				NUM_BANKS(ADDR_SURF_4_BANK));
1178
1179		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1180			WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1181		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1182			if (reg_offset != 7)
1183				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1184		break;
1185	case CHIP_HAWAII:
1186		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1187			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1188			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1189			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1190		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1191			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1192			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1193			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1194		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1195			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1196			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1197			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1198		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1199			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1200			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1201			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1202		tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1203			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1204			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1205			   TILE_SPLIT(split_equal_to_row_size));
1206		tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1207			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1208			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1209			   TILE_SPLIT(split_equal_to_row_size));
1210		tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1211			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1212			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1213			   TILE_SPLIT(split_equal_to_row_size));
1214		tile[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1215			   PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1216			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1217			   TILE_SPLIT(split_equal_to_row_size));
1218		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1219			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
1220		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1221			   PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1222			   MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1223		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1224			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1225			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1226			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1227		tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1228			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1229			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1230			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1231		tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1232			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1233			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1234			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1235		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1236			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1237			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1238		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1239			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1240			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1241			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1242		tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1243			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1244			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1245			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1246		tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1247			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1248			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1249			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1250		tile[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1251			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1252			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1253			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1254		tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1255			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1256			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1257			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1258		tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1259			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1260			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1261		tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1262			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1263			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1264			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1265		tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1266			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1267			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1268			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1269		tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1270			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1271			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1272			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1273		tile[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1274			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1275			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1276			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1277		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1278			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1279			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1280			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1281		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1282			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1283			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1284			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1285		tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1286			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1287			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1288			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1289		tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1290			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1291			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1292		tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1293			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1294			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1295			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1296		tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1297			    PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1298			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1299			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1300		tile[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1301			    PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1302			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1303			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1304
1305		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1306				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1307				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1308				NUM_BANKS(ADDR_SURF_16_BANK));
1309		macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1310				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1311				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1312				NUM_BANKS(ADDR_SURF_16_BANK));
1313		macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1314				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1315				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1316				NUM_BANKS(ADDR_SURF_16_BANK));
1317		macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1318				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1319				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1320				NUM_BANKS(ADDR_SURF_16_BANK));
1321		macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1322				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1323				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1324				NUM_BANKS(ADDR_SURF_8_BANK));
1325		macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1326				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1327				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1328				NUM_BANKS(ADDR_SURF_4_BANK));
1329		macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1330				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1331				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1332				NUM_BANKS(ADDR_SURF_4_BANK));
1333		macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1334				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1335				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1336				NUM_BANKS(ADDR_SURF_16_BANK));
1337		macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1338				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1339				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1340				NUM_BANKS(ADDR_SURF_16_BANK));
1341		macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1342				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1343				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1344				NUM_BANKS(ADDR_SURF_16_BANK));
1345		macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1346				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1347				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1348				NUM_BANKS(ADDR_SURF_8_BANK));
1349		macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1350				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1351				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1352				NUM_BANKS(ADDR_SURF_16_BANK));
1353		macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1354				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1355				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1356				NUM_BANKS(ADDR_SURF_8_BANK));
1357		macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1358				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1359				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1360				NUM_BANKS(ADDR_SURF_4_BANK));
1361
1362		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1363			WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1364		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1365			if (reg_offset != 7)
1366				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1367		break;
1368	case CHIP_KABINI:
1369	case CHIP_KAVERI:
1370	case CHIP_MULLINS:
1371	default:
1372		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1373			   PIPE_CONFIG(ADDR_SURF_P2) |
1374			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1375			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1376		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1377			   PIPE_CONFIG(ADDR_SURF_P2) |
1378			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1379			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1380		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1381			   PIPE_CONFIG(ADDR_SURF_P2) |
1382			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1383			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1384		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1385			   PIPE_CONFIG(ADDR_SURF_P2) |
1386			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1387			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1388		tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1389			   PIPE_CONFIG(ADDR_SURF_P2) |
1390			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1391			   TILE_SPLIT(split_equal_to_row_size));
1392		tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1393			   PIPE_CONFIG(ADDR_SURF_P2) |
1394			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1395		tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1396			   PIPE_CONFIG(ADDR_SURF_P2) |
1397			   MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1398			   TILE_SPLIT(split_equal_to_row_size));
1399		tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1400		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1401			   PIPE_CONFIG(ADDR_SURF_P2));
1402		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1403			   PIPE_CONFIG(ADDR_SURF_P2) |
1404			   MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1405		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1406			    PIPE_CONFIG(ADDR_SURF_P2) |
1407			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1408			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1409		tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1410			    PIPE_CONFIG(ADDR_SURF_P2) |
1411			    MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1412			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1413		tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1414		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1415			    PIPE_CONFIG(ADDR_SURF_P2) |
1416			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1417		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1418			    PIPE_CONFIG(ADDR_SURF_P2) |
1419			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1420			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1421		tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1422			    PIPE_CONFIG(ADDR_SURF_P2) |
1423			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1424			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1425		tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1426			    PIPE_CONFIG(ADDR_SURF_P2) |
1427			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1428			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1429		tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1430		tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1431			    PIPE_CONFIG(ADDR_SURF_P2) |
1432			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1433			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1434		tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1435			    PIPE_CONFIG(ADDR_SURF_P2) |
1436			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1437		tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1438			    PIPE_CONFIG(ADDR_SURF_P2) |
1439			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1440			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1441		tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1442			    PIPE_CONFIG(ADDR_SURF_P2) |
1443			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1444			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1445		tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1446			    PIPE_CONFIG(ADDR_SURF_P2) |
1447			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1448			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1449		tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1450		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1451			    PIPE_CONFIG(ADDR_SURF_P2) |
1452			    MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1453			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1454		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1455			    PIPE_CONFIG(ADDR_SURF_P2) |
1456			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1457			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1458		tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1459			    PIPE_CONFIG(ADDR_SURF_P2) |
1460			    MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1461			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1462		tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1463			    PIPE_CONFIG(ADDR_SURF_P2) |
1464			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1465		tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1466			    PIPE_CONFIG(ADDR_SURF_P2) |
1467			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1468			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1469		tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1470			    PIPE_CONFIG(ADDR_SURF_P2) |
1471			    MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1472			    SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1473		tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1474
1475		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1476				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1477				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1478				NUM_BANKS(ADDR_SURF_8_BANK));
1479		macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1480				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1481				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1482				NUM_BANKS(ADDR_SURF_8_BANK));
1483		macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1484				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1485				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1486				NUM_BANKS(ADDR_SURF_8_BANK));
1487		macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1488				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1489				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1490				NUM_BANKS(ADDR_SURF_8_BANK));
1491		macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1492				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1493				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1494				NUM_BANKS(ADDR_SURF_8_BANK));
1495		macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1496				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1497				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1498				NUM_BANKS(ADDR_SURF_8_BANK));
1499		macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1500				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1501				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1502				NUM_BANKS(ADDR_SURF_8_BANK));
1503		macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1504				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1505				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1506				NUM_BANKS(ADDR_SURF_16_BANK));
1507		macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1508				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1509				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1510				NUM_BANKS(ADDR_SURF_16_BANK));
1511		macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1512				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1513				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1514				NUM_BANKS(ADDR_SURF_16_BANK));
1515		macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1516				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1517				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1518				NUM_BANKS(ADDR_SURF_16_BANK));
1519		macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1520				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1521				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1522				NUM_BANKS(ADDR_SURF_16_BANK));
1523		macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1524				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1525				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1526				NUM_BANKS(ADDR_SURF_16_BANK));
1527		macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1528				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1529				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1530				NUM_BANKS(ADDR_SURF_8_BANK));
1531
1532		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1533			WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1534		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1535			if (reg_offset != 7)
1536				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1537		break;
1538	}
1539}
1540
1541/**
1542 * gfx_v7_0_select_se_sh - select which SE, SH to address
1543 *
1544 * @adev: amdgpu_device pointer
1545 * @se_num: shader engine to address
1546 * @sh_num: sh block to address
1547 * @instance: Certain registers are instanced per SE or SH.
1548 *            0xffffffff means broadcast to all SEs or SHs (CIK).
1549 * @xcc_id: xcc accelerated compute core id
1550 * Select which SE, SH combinations to address.
1551 */
1552static void gfx_v7_0_select_se_sh(struct amdgpu_device *adev,
1553				  u32 se_num, u32 sh_num, u32 instance,
1554				  int xcc_id)
1555{
1556	u32 data;
1557
1558	if (instance == 0xffffffff)
1559		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1560	else
1561		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
1562
1563	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1564		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1565			GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
1566	else if (se_num == 0xffffffff)
1567		data |= GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK |
1568			(sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT);
1569	else if (sh_num == 0xffffffff)
1570		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1571			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1572	else
1573		data |= (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT) |
1574			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1575	WREG32(mmGRBM_GFX_INDEX, data);
1576}
1577
1578/**
1579 * gfx_v7_0_get_rb_active_bitmap - computes the mask of enabled RBs
1580 *
1581 * @adev: amdgpu_device pointer
1582 *
1583 * Calculates the bitmask of enabled RBs (CIK).
1584 * Returns the enabled RB bitmask.
1585 */
1586static u32 gfx_v7_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1587{
1588	u32 data, mask;
1589
1590	data = RREG32(mmCC_RB_BACKEND_DISABLE);
1591	data |= RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1592
1593	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1594	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1595
1596	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
1597					 adev->gfx.config.max_sh_per_se);
1598
1599	return (~data) & mask;
1600}
1601
1602static void
1603gfx_v7_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
1604{
1605	switch (adev->asic_type) {
1606	case CHIP_BONAIRE:
1607		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
1608			  SE_XSEL(1) | SE_YSEL(1);
1609		*rconf1 |= 0x0;
1610		break;
1611	case CHIP_HAWAII:
1612		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
1613			  RB_XSEL2(1) | PKR_MAP(2) | PKR_XSEL(1) |
1614			  PKR_YSEL(1) | SE_MAP(2) | SE_XSEL(2) |
1615			  SE_YSEL(3);
1616		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
1617			   SE_PAIR_YSEL(2);
1618		break;
1619	case CHIP_KAVERI:
1620		*rconf |= RB_MAP_PKR0(2);
1621		*rconf1 |= 0x0;
1622		break;
1623	case CHIP_KABINI:
1624	case CHIP_MULLINS:
1625		*rconf |= 0x0;
1626		*rconf1 |= 0x0;
1627		break;
1628	default:
1629		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
1630		break;
1631	}
1632}
1633
1634static void
1635gfx_v7_0_write_harvested_raster_configs(struct amdgpu_device *adev,
1636					u32 raster_config, u32 raster_config_1,
1637					unsigned rb_mask, unsigned num_rb)
1638{
1639	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
1640	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
1641	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
1642	unsigned rb_per_se = num_rb / num_se;
1643	unsigned se_mask[4];
1644	unsigned se;
1645
1646	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
1647	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
1648	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
1649	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
1650
1651	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
1652	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
1653	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
1654
1655	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
1656			     (!se_mask[2] && !se_mask[3]))) {
1657		raster_config_1 &= ~SE_PAIR_MAP_MASK;
1658
1659		if (!se_mask[0] && !se_mask[1]) {
1660			raster_config_1 |=
1661				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
1662		} else {
1663			raster_config_1 |=
1664				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
1665		}
1666	}
1667
1668	for (se = 0; se < num_se; se++) {
1669		unsigned raster_config_se = raster_config;
1670		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
1671		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
1672		int idx = (se / 2) * 2;
1673
1674		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
1675			raster_config_se &= ~SE_MAP_MASK;
1676
1677			if (!se_mask[idx]) {
1678				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
1679			} else {
1680				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
1681			}
1682		}
1683
1684		pkr0_mask &= rb_mask;
1685		pkr1_mask &= rb_mask;
1686		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
1687			raster_config_se &= ~PKR_MAP_MASK;
1688
1689			if (!pkr0_mask) {
1690				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
1691			} else {
1692				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
1693			}
1694		}
1695
1696		if (rb_per_se >= 2) {
1697			unsigned rb0_mask = 1 << (se * rb_per_se);
1698			unsigned rb1_mask = rb0_mask << 1;
1699
1700			rb0_mask &= rb_mask;
1701			rb1_mask &= rb_mask;
1702			if (!rb0_mask || !rb1_mask) {
1703				raster_config_se &= ~RB_MAP_PKR0_MASK;
1704
1705				if (!rb0_mask) {
1706					raster_config_se |=
1707						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
1708				} else {
1709					raster_config_se |=
1710						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
1711				}
1712			}
1713
1714			if (rb_per_se > 2) {
1715				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
1716				rb1_mask = rb0_mask << 1;
1717				rb0_mask &= rb_mask;
1718				rb1_mask &= rb_mask;
1719				if (!rb0_mask || !rb1_mask) {
1720					raster_config_se &= ~RB_MAP_PKR1_MASK;
1721
1722					if (!rb0_mask) {
1723						raster_config_se |=
1724							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
1725					} else {
1726						raster_config_se |=
1727							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
1728					}
1729				}
1730			}
1731		}
1732
1733		/* GRBM_GFX_INDEX has a different offset on CI+ */
1734		gfx_v7_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0);
1735		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
1736		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1737	}
1738
1739	/* GRBM_GFX_INDEX has a different offset on CI+ */
1740	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1741}
1742
1743/**
1744 * gfx_v7_0_setup_rb - setup the RBs on the asic
1745 *
1746 * @adev: amdgpu_device pointer
1747 *
1748 * Configures per-SE/SH RB registers (CIK).
1749 */
1750static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
1751{
1752	int i, j;
1753	u32 data;
1754	u32 raster_config = 0, raster_config_1 = 0;
1755	u32 active_rbs = 0;
1756	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1757					adev->gfx.config.max_sh_per_se;
1758	unsigned num_rb_pipes;
1759
1760	mutex_lock(&adev->grbm_idx_mutex);
1761	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1762		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1763			gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff, 0);
1764			data = gfx_v7_0_get_rb_active_bitmap(adev);
1765			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1766					       rb_bitmap_width_per_sh);
1767		}
1768	}
1769	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1770
1771	adev->gfx.config.backend_enable_mask = active_rbs;
1772	adev->gfx.config.num_rbs = hweight32(active_rbs);
1773
1774	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
1775			     adev->gfx.config.max_shader_engines, 16);
1776
1777	gfx_v7_0_raster_config(adev, &raster_config, &raster_config_1);
1778
1779	if (!adev->gfx.config.backend_enable_mask ||
1780			adev->gfx.config.num_rbs >= num_rb_pipes) {
1781		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
1782		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1783	} else {
1784		gfx_v7_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
1785							adev->gfx.config.backend_enable_mask,
1786							num_rb_pipes);
1787	}
1788
1789	/* cache the values for userspace */
1790	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1791		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1792			gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff, 0);
1793			adev->gfx.config.rb_config[i][j].rb_backend_disable =
1794				RREG32(mmCC_RB_BACKEND_DISABLE);
1795			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
1796				RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1797			adev->gfx.config.rb_config[i][j].raster_config =
1798				RREG32(mmPA_SC_RASTER_CONFIG);
1799			adev->gfx.config.rb_config[i][j].raster_config_1 =
1800				RREG32(mmPA_SC_RASTER_CONFIG_1);
1801		}
1802	}
1803	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1804	mutex_unlock(&adev->grbm_idx_mutex);
1805}
1806
1807#define DEFAULT_SH_MEM_BASES	(0x6000)
1808/**
1809 * gfx_v7_0_init_compute_vmid - gart enable
1810 *
1811 * @adev: amdgpu_device pointer
1812 *
1813 * Initialize compute vmid sh_mem registers
1814 *
1815 */
1816static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
1817{
1818	int i;
1819	uint32_t sh_mem_config;
1820	uint32_t sh_mem_bases;
1821
1822	/*
1823	 * Configure apertures:
1824	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1825	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1826	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1827	*/
1828	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1829	sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1830			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1831	sh_mem_config |= MTYPE_NONCACHED << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT;
1832	mutex_lock(&adev->srbm_mutex);
1833	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1834		cik_srbm_select(adev, 0, 0, 0, i);
1835		/* CP and shaders */
1836		WREG32(mmSH_MEM_CONFIG, sh_mem_config);
1837		WREG32(mmSH_MEM_APE1_BASE, 1);
1838		WREG32(mmSH_MEM_APE1_LIMIT, 0);
1839		WREG32(mmSH_MEM_BASES, sh_mem_bases);
1840	}
1841	cik_srbm_select(adev, 0, 0, 0, 0);
1842	mutex_unlock(&adev->srbm_mutex);
1843
1844	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
1845	   access. These should be enabled by FW for target VMIDs. */
1846	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1847		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
1848		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
1849		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
1850		WREG32(amdgpu_gds_reg_offset[i].oa, 0);
1851	}
1852}
1853
1854static void gfx_v7_0_init_gds_vmid(struct amdgpu_device *adev)
1855{
1856	int vmid;
1857
1858	/*
1859	 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1860	 * access. Compute VMIDs should be enabled by FW for target VMIDs,
1861	 * the driver can enable them for graphics. VMID0 should maintain
1862	 * access so that HWS firmware can save/restore entries.
1863	 */
1864	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
1865		WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0);
1866		WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0);
1867		WREG32(amdgpu_gds_reg_offset[vmid].gws, 0);
1868		WREG32(amdgpu_gds_reg_offset[vmid].oa, 0);
1869	}
1870}
1871
1872static void gfx_v7_0_config_init(struct amdgpu_device *adev)
1873{
1874	adev->gfx.config.double_offchip_lds_buf = 1;
1875}
1876
1877/**
1878 * gfx_v7_0_constants_init - setup the 3D engine
1879 *
1880 * @adev: amdgpu_device pointer
1881 *
1882 * init the gfx constants such as the 3D engine, tiling configuration
1883 * registers, maximum number of quad pipes, render backends...
1884 */
1885static void gfx_v7_0_constants_init(struct amdgpu_device *adev)
1886{
1887	u32 sh_mem_cfg, sh_static_mem_cfg, sh_mem_base;
1888	u32 tmp;
1889	int i;
1890
1891	WREG32(mmGRBM_CNTL, (0xff << GRBM_CNTL__READ_TIMEOUT__SHIFT));
1892
1893	WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1894	WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1895	WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
1896
1897	gfx_v7_0_tiling_mode_table_init(adev);
1898
1899	gfx_v7_0_setup_rb(adev);
1900	gfx_v7_0_get_cu_info(adev);
1901	gfx_v7_0_config_init(adev);
1902
1903	/* set HW defaults for 3D engine */
1904	WREG32(mmCP_MEQ_THRESHOLDS,
1905	       (0x30 << CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT) |
1906	       (0x60 << CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT));
1907
1908	mutex_lock(&adev->grbm_idx_mutex);
1909	/*
1910	 * making sure that the following register writes will be broadcasted
1911	 * to all the shaders
1912	 */
1913	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1914
1915	/* XXX SH_MEM regs */
1916	/* where to put LDS, scratch, GPUVM in FSA64 space */
1917	sh_mem_cfg = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1918				   SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1919	sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, DEFAULT_MTYPE,
1920				   MTYPE_NC);
1921	sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, APE1_MTYPE,
1922				   MTYPE_UC);
1923	sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, PRIVATE_ATC, 0);
1924
1925	sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
1926				   SWIZZLE_ENABLE, 1);
1927	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1928				   ELEMENT_SIZE, 1);
1929	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1930				   INDEX_STRIDE, 3);
1931	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
1932
1933	mutex_lock(&adev->srbm_mutex);
1934	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
1935		if (i == 0)
1936			sh_mem_base = 0;
1937		else
1938			sh_mem_base = adev->gmc.shared_aperture_start >> 48;
1939		cik_srbm_select(adev, 0, 0, 0, i);
1940		/* CP and shaders */
1941		WREG32(mmSH_MEM_CONFIG, sh_mem_cfg);
1942		WREG32(mmSH_MEM_APE1_BASE, 1);
1943		WREG32(mmSH_MEM_APE1_LIMIT, 0);
1944		WREG32(mmSH_MEM_BASES, sh_mem_base);
1945	}
1946	cik_srbm_select(adev, 0, 0, 0, 0);
1947	mutex_unlock(&adev->srbm_mutex);
1948
1949	gfx_v7_0_init_compute_vmid(adev);
1950	gfx_v7_0_init_gds_vmid(adev);
1951
1952	WREG32(mmSX_DEBUG_1, 0x20);
1953
1954	WREG32(mmTA_CNTL_AUX, 0x00010000);
1955
1956	tmp = RREG32(mmSPI_CONFIG_CNTL);
1957	tmp |= 0x03000000;
1958	WREG32(mmSPI_CONFIG_CNTL, tmp);
1959
1960	WREG32(mmSQ_CONFIG, 1);
1961
1962	WREG32(mmDB_DEBUG, 0);
1963
1964	tmp = RREG32(mmDB_DEBUG2) & ~0xf00fffff;
1965	tmp |= 0x00000400;
1966	WREG32(mmDB_DEBUG2, tmp);
1967
1968	tmp = RREG32(mmDB_DEBUG3) & ~0x0002021c;
1969	tmp |= 0x00020200;
1970	WREG32(mmDB_DEBUG3, tmp);
1971
1972	tmp = RREG32(mmCB_HW_CONTROL) & ~0x00010000;
1973	tmp |= 0x00018208;
1974	WREG32(mmCB_HW_CONTROL, tmp);
1975
1976	WREG32(mmSPI_CONFIG_CNTL_1, (4 << SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT));
1977
1978	WREG32(mmPA_SC_FIFO_SIZE,
1979		((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1980		(adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1981		(adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1982		(adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)));
1983
1984	WREG32(mmVGT_NUM_INSTANCES, 1);
1985
1986	WREG32(mmCP_PERFMON_CNTL, 0);
1987
1988	WREG32(mmSQ_CONFIG, 0);
1989
1990	WREG32(mmPA_SC_FORCE_EOV_MAX_CNTS,
1991		((4095 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT) |
1992		(255 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT)));
1993
1994	WREG32(mmVGT_CACHE_INVALIDATION,
1995		(VC_AND_TC << VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT) |
1996		(ES_AND_GS_AUTO << VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT));
1997
1998	WREG32(mmVGT_GS_VERTEX_REUSE, 16);
1999	WREG32(mmPA_SC_LINE_STIPPLE_STATE, 0);
2000
2001	WREG32(mmPA_CL_ENHANCE, PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK |
2002			(3 << PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT));
2003	WREG32(mmPA_SC_ENHANCE, PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK);
2004
2005	tmp = RREG32(mmSPI_ARB_PRIORITY);
2006	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
2007	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
2008	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
2009	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
2010	WREG32(mmSPI_ARB_PRIORITY, tmp);
2011
2012	mutex_unlock(&adev->grbm_idx_mutex);
2013
2014	udelay(50);
2015}
2016
2017/**
2018 * gfx_v7_0_ring_test_ring - basic gfx ring test
2019 *
2020 * @ring: amdgpu_ring structure holding ring information
2021 *
2022 * Allocate a scratch register and write to it using the gfx ring (CIK).
2023 * Provides a basic gfx ring test to verify that the ring is working.
2024 * Used by gfx_v7_0_cp_gfx_resume();
2025 * Returns 0 on success, error on failure.
2026 */
2027static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
2028{
2029	struct amdgpu_device *adev = ring->adev;
2030	uint32_t tmp = 0;
2031	unsigned i;
2032	int r;
2033
2034	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
2035	r = amdgpu_ring_alloc(ring, 3);
2036	if (r)
2037		return r;
2038
2039	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2040	amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START);
2041	amdgpu_ring_write(ring, 0xDEADBEEF);
2042	amdgpu_ring_commit(ring);
2043
2044	for (i = 0; i < adev->usec_timeout; i++) {
2045		tmp = RREG32(mmSCRATCH_REG0);
2046		if (tmp == 0xDEADBEEF)
2047			break;
2048		udelay(1);
2049	}
2050	if (i >= adev->usec_timeout)
2051		r = -ETIMEDOUT;
2052	return r;
2053}
2054
2055/**
2056 * gfx_v7_0_ring_emit_hdp_flush - emit an hdp flush on the cp
2057 *
2058 * @ring: amdgpu_ring structure holding ring information
2059 *
2060 * Emits an hdp flush on the cp.
2061 */
2062static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2063{
2064	u32 ref_and_mask;
2065	int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
2066
2067	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
2068		switch (ring->me) {
2069		case 1:
2070			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
2071			break;
2072		case 2:
2073			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
2074			break;
2075		default:
2076			return;
2077		}
2078	} else {
2079		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
2080	}
2081
2082	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2083	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
2084				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
2085				 WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
2086	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
2087	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
2088	amdgpu_ring_write(ring, ref_and_mask);
2089	amdgpu_ring_write(ring, ref_and_mask);
2090	amdgpu_ring_write(ring, 0x20); /* poll interval */
2091}
2092
2093static void gfx_v7_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
2094{
2095	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2096	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
2097		EVENT_INDEX(4));
2098
2099	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2100	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
2101		EVENT_INDEX(0));
2102}
2103
2104/**
2105 * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring
2106 *
2107 * @ring: amdgpu_ring structure holding ring information
2108 * @addr: address
2109 * @seq: sequence number
2110 * @flags: fence related flags
2111 *
2112 * Emits a fence sequence number on the gfx ring and flushes
2113 * GPU caches.
2114 */
2115static void gfx_v7_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
2116					 u64 seq, unsigned flags)
2117{
2118	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2119	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2120	/* Workaround for cache flush problems. First send a dummy EOP
2121	 * event down the pipe with seq one below.
2122	 */
2123	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2124	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2125				 EOP_TC_ACTION_EN |
2126				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2127				 EVENT_INDEX(5)));
2128	amdgpu_ring_write(ring, addr & 0xfffffffc);
2129	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2130				DATA_SEL(1) | INT_SEL(0));
2131	amdgpu_ring_write(ring, lower_32_bits(seq - 1));
2132	amdgpu_ring_write(ring, upper_32_bits(seq - 1));
2133
2134	/* Then send the real EOP event down the pipe. */
2135	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2136	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2137				 EOP_TC_ACTION_EN |
2138				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2139				 EVENT_INDEX(5)));
2140	amdgpu_ring_write(ring, addr & 0xfffffffc);
2141	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2142				DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2143	amdgpu_ring_write(ring, lower_32_bits(seq));
2144	amdgpu_ring_write(ring, upper_32_bits(seq));
2145}
2146
2147/**
2148 * gfx_v7_0_ring_emit_fence_compute - emit a fence on the compute ring
2149 *
2150 * @ring: amdgpu_ring structure holding ring information
2151 * @addr: address
2152 * @seq: sequence number
2153 * @flags: fence related flags
2154 *
2155 * Emits a fence sequence number on the compute ring and flushes
2156 * GPU caches.
2157 */
2158static void gfx_v7_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
2159					     u64 addr, u64 seq,
2160					     unsigned flags)
2161{
2162	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2163	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2164
2165	/* RELEASE_MEM - flush caches, send int */
2166	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
2167	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2168				 EOP_TC_ACTION_EN |
2169				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2170				 EVENT_INDEX(5)));
2171	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2172	amdgpu_ring_write(ring, addr & 0xfffffffc);
2173	amdgpu_ring_write(ring, upper_32_bits(addr));
2174	amdgpu_ring_write(ring, lower_32_bits(seq));
2175	amdgpu_ring_write(ring, upper_32_bits(seq));
2176}
2177
2178/*
2179 * IB stuff
2180 */
2181/**
2182 * gfx_v7_0_ring_emit_ib_gfx - emit an IB (Indirect Buffer) on the ring
2183 *
2184 * @ring: amdgpu_ring structure holding ring information
2185 * @job: job to retrieve vmid from
2186 * @ib: amdgpu indirect buffer object
2187 * @flags: options (AMDGPU_HAVE_CTX_SWITCH)
2188 *
2189 * Emits an DE (drawing engine) or CE (constant engine) IB
2190 * on the gfx ring.  IBs are usually generated by userspace
2191 * acceleration drivers and submitted to the kernel for
2192 * scheduling on the ring.  This function schedules the IB
2193 * on the gfx ring for execution by the GPU.
2194 */
2195static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
2196					struct amdgpu_job *job,
2197					struct amdgpu_ib *ib,
2198					uint32_t flags)
2199{
2200	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2201	u32 header, control = 0;
2202
2203	/* insert SWITCH_BUFFER packet before first IB in the ring frame */
2204	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
2205		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2206		amdgpu_ring_write(ring, 0);
2207	}
2208
2209	if (ib->flags & AMDGPU_IB_FLAG_CE)
2210		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2211	else
2212		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2213
2214	control |= ib->length_dw | (vmid << 24);
2215
2216	amdgpu_ring_write(ring, header);
2217	amdgpu_ring_write(ring,
2218#ifdef __BIG_ENDIAN
2219			  (2 << 0) |
2220#endif
2221			  (ib->gpu_addr & 0xFFFFFFFC));
2222	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2223	amdgpu_ring_write(ring, control);
2224}
2225
2226static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
2227					  struct amdgpu_job *job,
2228					  struct amdgpu_ib *ib,
2229					  uint32_t flags)
2230{
2231	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2232	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
2233
2234	/* Currently, there is a high possibility to get wave ID mismatch
2235	 * between ME and GDS, leading to a hw deadlock, because ME generates
2236	 * different wave IDs than the GDS expects. This situation happens
2237	 * randomly when at least 5 compute pipes use GDS ordered append.
2238	 * The wave IDs generated by ME are also wrong after suspend/resume.
2239	 * Those are probably bugs somewhere else in the kernel driver.
2240	 *
2241	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
2242	 * GDS to 0 for this ring (me/pipe).
2243	 */
2244	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
2245		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2246		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
2247		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
2248	}
2249
2250	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2251	amdgpu_ring_write(ring,
2252#ifdef __BIG_ENDIAN
2253					  (2 << 0) |
2254#endif
2255					  (ib->gpu_addr & 0xFFFFFFFC));
2256	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2257	amdgpu_ring_write(ring, control);
2258}
2259
2260static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
2261{
2262	uint32_t dw2 = 0;
2263
2264	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
2265	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
2266		gfx_v7_0_ring_emit_vgt_flush(ring);
2267		/* set load_global_config & load_global_uconfig */
2268		dw2 |= 0x8001;
2269		/* set load_cs_sh_regs */
2270		dw2 |= 0x01000000;
2271		/* set load_per_context_state & load_gfx_sh_regs */
2272		dw2 |= 0x10002;
2273	}
2274
2275	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2276	amdgpu_ring_write(ring, dw2);
2277	amdgpu_ring_write(ring, 0);
2278}
2279
2280/**
2281 * gfx_v7_0_ring_test_ib - basic ring IB test
2282 *
2283 * @ring: amdgpu_ring structure holding ring information
2284 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
2285 *
2286 * Allocate an IB and execute it on the gfx ring (CIK).
2287 * Provides a basic gfx ring test to verify that IBs are working.
2288 * Returns 0 on success, error on failure.
2289 */
2290static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
2291{
2292	struct amdgpu_device *adev = ring->adev;
2293	struct amdgpu_ib ib;
2294	struct dma_fence *f = NULL;
2295	uint32_t tmp = 0;
2296	long r;
2297
2298	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
2299	memset(&ib, 0, sizeof(ib));
2300	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
2301	if (r)
2302		return r;
2303
2304	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2305	ib.ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START;
2306	ib.ptr[2] = 0xDEADBEEF;
2307	ib.length_dw = 3;
2308
2309	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
2310	if (r)
2311		goto error;
2312
2313	r = dma_fence_wait_timeout(f, false, timeout);
2314	if (r == 0) {
2315		r = -ETIMEDOUT;
2316		goto error;
2317	} else if (r < 0) {
2318		goto error;
2319	}
2320	tmp = RREG32(mmSCRATCH_REG0);
2321	if (tmp == 0xDEADBEEF)
2322		r = 0;
2323	else
2324		r = -EINVAL;
2325
2326error:
2327	amdgpu_ib_free(adev, &ib, NULL);
2328	dma_fence_put(f);
2329	return r;
2330}
2331
2332/*
2333 * CP.
2334 * On CIK, gfx and compute now have independent command processors.
2335 *
2336 * GFX
2337 * Gfx consists of a single ring and can process both gfx jobs and
2338 * compute jobs.  The gfx CP consists of three microengines (ME):
2339 * PFP - Pre-Fetch Parser
2340 * ME - Micro Engine
2341 * CE - Constant Engine
2342 * The PFP and ME make up what is considered the Drawing Engine (DE).
2343 * The CE is an asynchronous engine used for updating buffer desciptors
2344 * used by the DE so that they can be loaded into cache in parallel
2345 * while the DE is processing state update packets.
2346 *
2347 * Compute
2348 * The compute CP consists of two microengines (ME):
2349 * MEC1 - Compute MicroEngine 1
2350 * MEC2 - Compute MicroEngine 2
2351 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
2352 * The queues are exposed to userspace and are programmed directly
2353 * by the compute runtime.
2354 */
2355/**
2356 * gfx_v7_0_cp_gfx_enable - enable/disable the gfx CP MEs
2357 *
2358 * @adev: amdgpu_device pointer
2359 * @enable: enable or disable the MEs
2360 *
2361 * Halts or unhalts the gfx MEs.
2362 */
2363static void gfx_v7_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2364{
2365	if (enable)
2366		WREG32(mmCP_ME_CNTL, 0);
2367	else
2368		WREG32(mmCP_ME_CNTL, (CP_ME_CNTL__ME_HALT_MASK |
2369				      CP_ME_CNTL__PFP_HALT_MASK |
2370				      CP_ME_CNTL__CE_HALT_MASK));
2371	udelay(50);
2372}
2373
2374/**
2375 * gfx_v7_0_cp_gfx_load_microcode - load the gfx CP ME ucode
2376 *
2377 * @adev: amdgpu_device pointer
2378 *
2379 * Loads the gfx PFP, ME, and CE ucode.
2380 * Returns 0 for success, -EINVAL if the ucode is not available.
2381 */
2382static int gfx_v7_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2383{
2384	const struct gfx_firmware_header_v1_0 *pfp_hdr;
2385	const struct gfx_firmware_header_v1_0 *ce_hdr;
2386	const struct gfx_firmware_header_v1_0 *me_hdr;
2387	const __le32 *fw_data;
2388	unsigned i, fw_size;
2389
2390	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2391		return -EINVAL;
2392
2393	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
2394	ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
2395	me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
2396
2397	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2398	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2399	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2400	adev->gfx.pfp_fw_version = le32_to_cpu(pfp_hdr->header.ucode_version);
2401	adev->gfx.ce_fw_version = le32_to_cpu(ce_hdr->header.ucode_version);
2402	adev->gfx.me_fw_version = le32_to_cpu(me_hdr->header.ucode_version);
2403	adev->gfx.me_feature_version = le32_to_cpu(me_hdr->ucode_feature_version);
2404	adev->gfx.ce_feature_version = le32_to_cpu(ce_hdr->ucode_feature_version);
2405	adev->gfx.pfp_feature_version = le32_to_cpu(pfp_hdr->ucode_feature_version);
2406
2407	gfx_v7_0_cp_gfx_enable(adev, false);
2408
2409	/* PFP */
2410	fw_data = (const __le32 *)
2411		(adev->gfx.pfp_fw->data +
2412		 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2413	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2414	WREG32(mmCP_PFP_UCODE_ADDR, 0);
2415	for (i = 0; i < fw_size; i++)
2416		WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2417	WREG32(mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2418
2419	/* CE */
2420	fw_data = (const __le32 *)
2421		(adev->gfx.ce_fw->data +
2422		 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2423	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
2424	WREG32(mmCP_CE_UCODE_ADDR, 0);
2425	for (i = 0; i < fw_size; i++)
2426		WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
2427	WREG32(mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
2428
2429	/* ME */
2430	fw_data = (const __le32 *)
2431		(adev->gfx.me_fw->data +
2432		 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2433	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
2434	WREG32(mmCP_ME_RAM_WADDR, 0);
2435	for (i = 0; i < fw_size; i++)
2436		WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
2437	WREG32(mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
2438
2439	return 0;
2440}
2441
2442/**
2443 * gfx_v7_0_cp_gfx_start - start the gfx ring
2444 *
2445 * @adev: amdgpu_device pointer
2446 *
2447 * Enables the ring and loads the clear state context and other
2448 * packets required to init the ring.
2449 * Returns 0 for success, error for failure.
2450 */
2451static int gfx_v7_0_cp_gfx_start(struct amdgpu_device *adev)
2452{
2453	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
2454	const struct cs_section_def *sect = NULL;
2455	const struct cs_extent_def *ext = NULL;
2456	int r, i;
2457
2458	/* init the CP */
2459	WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
2460	WREG32(mmCP_ENDIAN_SWAP, 0);
2461	WREG32(mmCP_DEVICE_ID, 1);
2462
2463	gfx_v7_0_cp_gfx_enable(adev, true);
2464
2465	r = amdgpu_ring_alloc(ring, gfx_v7_0_get_csb_size(adev) + 8);
2466	if (r) {
2467		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2468		return r;
2469	}
2470
2471	/* init the CE partitions.  CE only used for gfx on CIK */
2472	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2473	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2474	amdgpu_ring_write(ring, 0x8000);
2475	amdgpu_ring_write(ring, 0x8000);
2476
2477	/* clear state buffer */
2478	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2479	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2480
2481	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2482	amdgpu_ring_write(ring, 0x80000000);
2483	amdgpu_ring_write(ring, 0x80000000);
2484
2485	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2486		for (ext = sect->section; ext->extent != NULL; ++ext) {
2487			if (sect->id == SECT_CONTEXT) {
2488				amdgpu_ring_write(ring,
2489						  PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2490				amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2491				for (i = 0; i < ext->reg_count; i++)
2492					amdgpu_ring_write(ring, ext->extent[i]);
2493			}
2494		}
2495	}
2496
2497	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2498	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2499	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
2500	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
2501
2502	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2503	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2504
2505	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2506	amdgpu_ring_write(ring, 0);
2507
2508	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2509	amdgpu_ring_write(ring, 0x00000316);
2510	amdgpu_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2511	amdgpu_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2512
2513	amdgpu_ring_commit(ring);
2514
2515	return 0;
2516}
2517
2518/**
2519 * gfx_v7_0_cp_gfx_resume - setup the gfx ring buffer registers
2520 *
2521 * @adev: amdgpu_device pointer
2522 *
2523 * Program the location and size of the gfx ring buffer
2524 * and test it to make sure it's working.
2525 * Returns 0 for success, error for failure.
2526 */
2527static int gfx_v7_0_cp_gfx_resume(struct amdgpu_device *adev)
2528{
2529	struct amdgpu_ring *ring;
2530	u32 tmp;
2531	u32 rb_bufsz;
2532	u64 rb_addr, rptr_addr;
2533	int r;
2534
2535	WREG32(mmCP_SEM_WAIT_TIMER, 0x0);
2536	if (adev->asic_type != CHIP_HAWAII)
2537		WREG32(mmCP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2538
2539	/* Set the write pointer delay */
2540	WREG32(mmCP_RB_WPTR_DELAY, 0);
2541
2542	/* set the RB to use vmid 0 */
2543	WREG32(mmCP_RB_VMID, 0);
2544
2545	WREG32(mmSCRATCH_ADDR, 0);
2546
2547	/* ring 0 - compute and gfx */
2548	/* Set ring buffer size */
2549	ring = &adev->gfx.gfx_ring[0];
2550	rb_bufsz = order_base_2(ring->ring_size / 8);
2551	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2552#ifdef __BIG_ENDIAN
2553	tmp |= 2 << CP_RB0_CNTL__BUF_SWAP__SHIFT;
2554#endif
2555	WREG32(mmCP_RB0_CNTL, tmp);
2556
2557	/* Initialize the ring buffer's read and write pointers */
2558	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
2559	ring->wptr = 0;
2560	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2561
2562	/* set the wb address wether it's enabled or not */
2563	rptr_addr = ring->rptr_gpu_addr;
2564	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2565	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2566
2567	/* scratch register shadowing is no longer supported */
2568	WREG32(mmSCRATCH_UMSK, 0);
2569
2570	mdelay(1);
2571	WREG32(mmCP_RB0_CNTL, tmp);
2572
2573	rb_addr = ring->gpu_addr >> 8;
2574	WREG32(mmCP_RB0_BASE, rb_addr);
2575	WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2576
2577	/* start the ring */
2578	gfx_v7_0_cp_gfx_start(adev);
2579	r = amdgpu_ring_test_helper(ring);
2580	if (r)
2581		return r;
2582
2583	return 0;
2584}
2585
2586static u64 gfx_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
2587{
2588	return *ring->rptr_cpu_addr;
2589}
2590
2591static u64 gfx_v7_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
2592{
2593	struct amdgpu_device *adev = ring->adev;
2594
2595	return RREG32(mmCP_RB0_WPTR);
2596}
2597
2598static void gfx_v7_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2599{
2600	struct amdgpu_device *adev = ring->adev;
2601
2602	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2603	(void)RREG32(mmCP_RB0_WPTR);
2604}
2605
2606static u64 gfx_v7_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
2607{
2608	/* XXX check if swapping is necessary on BE */
2609	return *ring->wptr_cpu_addr;
2610}
2611
2612static void gfx_v7_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
2613{
2614	struct amdgpu_device *adev = ring->adev;
2615
2616	/* XXX check if swapping is necessary on BE */
2617	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
2618	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2619}
2620
2621/**
2622 * gfx_v7_0_cp_compute_enable - enable/disable the compute CP MEs
2623 *
2624 * @adev: amdgpu_device pointer
2625 * @enable: enable or disable the MEs
2626 *
2627 * Halts or unhalts the compute MEs.
2628 */
2629static void gfx_v7_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2630{
2631	if (enable)
2632		WREG32(mmCP_MEC_CNTL, 0);
2633	else
2634		WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK |
2635				       CP_MEC_CNTL__MEC_ME2_HALT_MASK));
2636	udelay(50);
2637}
2638
2639/**
2640 * gfx_v7_0_cp_compute_load_microcode - load the compute CP ME ucode
2641 *
2642 * @adev: amdgpu_device pointer
2643 *
2644 * Loads the compute MEC1&2 ucode.
2645 * Returns 0 for success, -EINVAL if the ucode is not available.
2646 */
2647static int gfx_v7_0_cp_compute_load_microcode(struct amdgpu_device *adev)
2648{
2649	const struct gfx_firmware_header_v1_0 *mec_hdr;
2650	const __le32 *fw_data;
2651	unsigned i, fw_size;
2652
2653	if (!adev->gfx.mec_fw)
2654		return -EINVAL;
2655
2656	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2657	amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2658	adev->gfx.mec_fw_version = le32_to_cpu(mec_hdr->header.ucode_version);
2659	adev->gfx.mec_feature_version = le32_to_cpu(
2660					mec_hdr->ucode_feature_version);
2661
2662	gfx_v7_0_cp_compute_enable(adev, false);
2663
2664	/* MEC1 */
2665	fw_data = (const __le32 *)
2666		(adev->gfx.mec_fw->data +
2667		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2668	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
2669	WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2670	for (i = 0; i < fw_size; i++)
2671		WREG32(mmCP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
2672	WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2673
2674	if (adev->asic_type == CHIP_KAVERI) {
2675		const struct gfx_firmware_header_v1_0 *mec2_hdr;
2676
2677		if (!adev->gfx.mec2_fw)
2678			return -EINVAL;
2679
2680		mec2_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
2681		amdgpu_ucode_print_gfx_hdr(&mec2_hdr->header);
2682		adev->gfx.mec2_fw_version = le32_to_cpu(mec2_hdr->header.ucode_version);
2683		adev->gfx.mec2_feature_version = le32_to_cpu(
2684				mec2_hdr->ucode_feature_version);
2685
2686		/* MEC2 */
2687		fw_data = (const __le32 *)
2688			(adev->gfx.mec2_fw->data +
2689			 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
2690		fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
2691		WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2692		for (i = 0; i < fw_size; i++)
2693			WREG32(mmCP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
2694		WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2695	}
2696
2697	return 0;
2698}
2699
2700/**
2701 * gfx_v7_0_cp_compute_fini - stop the compute queues
2702 *
2703 * @adev: amdgpu_device pointer
2704 *
2705 * Stop the compute queues and tear down the driver queue
2706 * info.
2707 */
2708static void gfx_v7_0_cp_compute_fini(struct amdgpu_device *adev)
2709{
2710	int i;
2711
2712	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2713		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
2714
2715		amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
2716	}
2717}
2718
2719static void gfx_v7_0_mec_fini(struct amdgpu_device *adev)
2720{
2721	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
2722}
2723
2724static int gfx_v7_0_mec_init(struct amdgpu_device *adev)
2725{
2726	int r;
2727	u32 *hpd;
2728	size_t mec_hpd_size;
2729
2730	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2731
2732	/* take ownership of the relevant compute queues */
2733	amdgpu_gfx_compute_queue_acquire(adev);
2734
2735	/* allocate space for ALL pipes (even the ones we don't own) */
2736	mec_hpd_size = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec
2737		* GFX7_MEC_HPD_SIZE * 2;
2738
2739	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
2740				      AMDGPU_GEM_DOMAIN_VRAM |
2741				      AMDGPU_GEM_DOMAIN_GTT,
2742				      &adev->gfx.mec.hpd_eop_obj,
2743				      &adev->gfx.mec.hpd_eop_gpu_addr,
2744				      (void **)&hpd);
2745	if (r) {
2746		dev_warn(adev->dev, "(%d) create, pin or map of HDP EOP bo failed\n", r);
2747		gfx_v7_0_mec_fini(adev);
2748		return r;
2749	}
2750
2751	/* clear memory.  Not sure if this is required or not */
2752	memset(hpd, 0, mec_hpd_size);
2753
2754	amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2755	amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2756
2757	return 0;
2758}
2759
2760struct hqd_registers {
 
2761	u32 cp_mqd_base_addr;
2762	u32 cp_mqd_base_addr_hi;
2763	u32 cp_hqd_active;
2764	u32 cp_hqd_vmid;
2765	u32 cp_hqd_persistent_state;
2766	u32 cp_hqd_pipe_priority;
2767	u32 cp_hqd_queue_priority;
2768	u32 cp_hqd_quantum;
2769	u32 cp_hqd_pq_base;
2770	u32 cp_hqd_pq_base_hi;
2771	u32 cp_hqd_pq_rptr;
2772	u32 cp_hqd_pq_rptr_report_addr;
2773	u32 cp_hqd_pq_rptr_report_addr_hi;
2774	u32 cp_hqd_pq_wptr_poll_addr;
2775	u32 cp_hqd_pq_wptr_poll_addr_hi;
2776	u32 cp_hqd_pq_doorbell_control;
2777	u32 cp_hqd_pq_wptr;
2778	u32 cp_hqd_pq_control;
2779	u32 cp_hqd_ib_base_addr;
2780	u32 cp_hqd_ib_base_addr_hi;
2781	u32 cp_hqd_ib_rptr;
2782	u32 cp_hqd_ib_control;
2783	u32 cp_hqd_iq_timer;
2784	u32 cp_hqd_iq_rptr;
2785	u32 cp_hqd_dequeue_request;
2786	u32 cp_hqd_dma_offload;
2787	u32 cp_hqd_sema_cmd;
2788	u32 cp_hqd_msg_type;
2789	u32 cp_hqd_atomic0_preop_lo;
2790	u32 cp_hqd_atomic0_preop_hi;
2791	u32 cp_hqd_atomic1_preop_lo;
2792	u32 cp_hqd_atomic1_preop_hi;
2793	u32 cp_hqd_hq_scheduler0;
2794	u32 cp_hqd_hq_scheduler1;
2795	u32 cp_mqd_control;
2796};
2797
2798static void gfx_v7_0_compute_pipe_init(struct amdgpu_device *adev,
2799				       int mec, int pipe)
2800{
2801	u64 eop_gpu_addr;
2802	u32 tmp;
2803	size_t eop_offset = (mec * adev->gfx.mec.num_pipe_per_mec + pipe)
2804			    * GFX7_MEC_HPD_SIZE * 2;
2805
2806	mutex_lock(&adev->srbm_mutex);
2807	eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + eop_offset;
2808
2809	cik_srbm_select(adev, mec + 1, pipe, 0, 0);
2810
2811	/* write the EOP addr */
2812	WREG32(mmCP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
2813	WREG32(mmCP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
2814
2815	/* set the VMID assigned */
2816	WREG32(mmCP_HPD_EOP_VMID, 0);
2817
2818	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2819	tmp = RREG32(mmCP_HPD_EOP_CONTROL);
2820	tmp &= ~CP_HPD_EOP_CONTROL__EOP_SIZE_MASK;
2821	tmp |= order_base_2(GFX7_MEC_HPD_SIZE / 8);
2822	WREG32(mmCP_HPD_EOP_CONTROL, tmp);
2823
2824	cik_srbm_select(adev, 0, 0, 0, 0);
2825	mutex_unlock(&adev->srbm_mutex);
2826}
2827
2828static int gfx_v7_0_mqd_deactivate(struct amdgpu_device *adev)
2829{
2830	int i;
2831
2832	/* disable the queue if it's active */
2833	if (RREG32(mmCP_HQD_ACTIVE) & 1) {
2834		WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
2835		for (i = 0; i < adev->usec_timeout; i++) {
2836			if (!(RREG32(mmCP_HQD_ACTIVE) & 1))
2837				break;
2838			udelay(1);
2839		}
2840
2841		if (i == adev->usec_timeout)
2842			return -ETIMEDOUT;
2843
2844		WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
2845		WREG32(mmCP_HQD_PQ_RPTR, 0);
2846		WREG32(mmCP_HQD_PQ_WPTR, 0);
2847	}
2848
2849	return 0;
2850}
2851
2852static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
2853			     struct cik_mqd *mqd,
2854			     uint64_t mqd_gpu_addr,
2855			     struct amdgpu_ring *ring)
2856{
2857	u64 hqd_gpu_addr;
2858	u64 wb_gpu_addr;
2859
2860	/* init the mqd struct */
2861	memset(mqd, 0, sizeof(struct cik_mqd));
2862
2863	mqd->header = 0xC0310800;
2864	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2865	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2866	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2867	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2868
2869	/* enable doorbell? */
2870	mqd->cp_hqd_pq_doorbell_control =
2871		RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
2872	if (ring->use_doorbell)
2873		mqd->cp_hqd_pq_doorbell_control |= CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2874	else
2875		mqd->cp_hqd_pq_doorbell_control &= ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2876
2877	/* set the pointer to the MQD */
2878	mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
2879	mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
2880
2881	/* set MQD vmid to 0 */
2882	mqd->cp_mqd_control = RREG32(mmCP_MQD_CONTROL);
2883	mqd->cp_mqd_control &= ~CP_MQD_CONTROL__VMID_MASK;
2884
2885	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2886	hqd_gpu_addr = ring->gpu_addr >> 8;
2887	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2888	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2889
2890	/* set up the HQD, this is similar to CP_RB0_CNTL */
2891	mqd->cp_hqd_pq_control = RREG32(mmCP_HQD_PQ_CONTROL);
2892	mqd->cp_hqd_pq_control &=
2893		~(CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK |
2894				CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK);
2895
2896	mqd->cp_hqd_pq_control |=
2897		order_base_2(ring->ring_size / 8);
2898	mqd->cp_hqd_pq_control |=
2899		(order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8);
2900#ifdef __BIG_ENDIAN
2901	mqd->cp_hqd_pq_control |=
2902		2 << CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT;
2903#endif
2904	mqd->cp_hqd_pq_control &=
2905		~(CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK |
2906				CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK |
2907				CP_HQD_PQ_CONTROL__PQ_VOLATILE_MASK);
2908	mqd->cp_hqd_pq_control |=
2909		CP_HQD_PQ_CONTROL__PRIV_STATE_MASK |
2910		CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK; /* assuming kernel queue control */
2911
2912	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2913	wb_gpu_addr = ring->wptr_gpu_addr;
2914	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2915	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2916
2917	/* set the wb address wether it's enabled or not */
2918	wb_gpu_addr = ring->rptr_gpu_addr;
2919	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2920	mqd->cp_hqd_pq_rptr_report_addr_hi =
2921		upper_32_bits(wb_gpu_addr) & 0xffff;
2922
2923	/* enable the doorbell if requested */
2924	if (ring->use_doorbell) {
2925		mqd->cp_hqd_pq_doorbell_control =
2926			RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
2927		mqd->cp_hqd_pq_doorbell_control &=
2928			~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
2929		mqd->cp_hqd_pq_doorbell_control |=
2930			(ring->doorbell_index <<
2931			 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT);
2932		mqd->cp_hqd_pq_doorbell_control |=
2933			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2934		mqd->cp_hqd_pq_doorbell_control &=
2935			~(CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK |
2936					CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK);
2937
2938	} else {
2939		mqd->cp_hqd_pq_doorbell_control = 0;
2940	}
2941
2942	/* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2943	ring->wptr = 0;
2944	mqd->cp_hqd_pq_wptr = lower_32_bits(ring->wptr);
2945	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
2946
2947	/* set the vmid for the queue */
2948	mqd->cp_hqd_vmid = 0;
2949
2950	/* defaults */
2951	mqd->cp_hqd_ib_control = RREG32(mmCP_HQD_IB_CONTROL);
2952	mqd->cp_hqd_ib_base_addr_lo = RREG32(mmCP_HQD_IB_BASE_ADDR);
2953	mqd->cp_hqd_ib_base_addr_hi = RREG32(mmCP_HQD_IB_BASE_ADDR_HI);
2954	mqd->cp_hqd_ib_rptr = RREG32(mmCP_HQD_IB_RPTR);
2955	mqd->cp_hqd_persistent_state = RREG32(mmCP_HQD_PERSISTENT_STATE);
2956	mqd->cp_hqd_sema_cmd = RREG32(mmCP_HQD_SEMA_CMD);
2957	mqd->cp_hqd_msg_type = RREG32(mmCP_HQD_MSG_TYPE);
2958	mqd->cp_hqd_atomic0_preop_lo = RREG32(mmCP_HQD_ATOMIC0_PREOP_LO);
2959	mqd->cp_hqd_atomic0_preop_hi = RREG32(mmCP_HQD_ATOMIC0_PREOP_HI);
2960	mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO);
2961	mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI);
2962	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
2963	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
2964	mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
2965	mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
2966	mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR);
2967
2968	/* activate the queue */
2969	mqd->cp_hqd_active = 1;
2970}
2971
2972static int gfx_v7_0_mqd_commit(struct amdgpu_device *adev, struct cik_mqd *mqd)
2973{
2974	uint32_t tmp;
2975	uint32_t mqd_reg;
2976	uint32_t *mqd_data;
2977
2978	/* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_MQD_CONTROL */
2979	mqd_data = &mqd->cp_mqd_base_addr_lo;
2980
2981	/* disable wptr polling */
2982	tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
2983	tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
2984	WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
2985
2986	/* program all HQD registers */
2987	for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_MQD_CONTROL; mqd_reg++)
2988		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
2989
2990	/* activate the HQD */
2991	for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
2992		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
2993
2994	return 0;
2995}
2996
2997static int gfx_v7_0_compute_queue_init(struct amdgpu_device *adev, int ring_id)
2998{
2999	int r;
3000	u64 mqd_gpu_addr;
3001	struct cik_mqd *mqd;
3002	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
3003
3004	r = amdgpu_bo_create_reserved(adev, sizeof(struct cik_mqd), PAGE_SIZE,
3005				      AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
3006				      &mqd_gpu_addr, (void **)&mqd);
3007	if (r) {
3008		dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
3009		return r;
3010	}
3011
3012	mutex_lock(&adev->srbm_mutex);
3013	cik_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3014
3015	gfx_v7_0_mqd_init(adev, mqd, mqd_gpu_addr, ring);
3016	gfx_v7_0_mqd_deactivate(adev);
3017	gfx_v7_0_mqd_commit(adev, mqd);
3018
3019	cik_srbm_select(adev, 0, 0, 0, 0);
3020	mutex_unlock(&adev->srbm_mutex);
3021
3022	amdgpu_bo_kunmap(ring->mqd_obj);
3023	amdgpu_bo_unreserve(ring->mqd_obj);
3024	return 0;
3025}
3026
3027/**
3028 * gfx_v7_0_cp_compute_resume - setup the compute queue registers
3029 *
3030 * @adev: amdgpu_device pointer
3031 *
3032 * Program the compute queues and test them to make sure they
3033 * are working.
3034 * Returns 0 for success, error for failure.
3035 */
3036static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
3037{
3038	int r, i, j;
3039	u32 tmp;
3040	struct amdgpu_ring *ring;
3041
3042	/* fix up chicken bits */
3043	tmp = RREG32(mmCP_CPF_DEBUG);
3044	tmp |= (1 << 23);
3045	WREG32(mmCP_CPF_DEBUG, tmp);
3046
3047	/* init all pipes (even the ones we don't own) */
3048	for (i = 0; i < adev->gfx.mec.num_mec; i++)
3049		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++)
3050			gfx_v7_0_compute_pipe_init(adev, i, j);
3051
3052	/* init the queues */
3053	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3054		r = gfx_v7_0_compute_queue_init(adev, i);
3055		if (r) {
3056			gfx_v7_0_cp_compute_fini(adev);
3057			return r;
3058		}
3059	}
3060
3061	gfx_v7_0_cp_compute_enable(adev, true);
3062
3063	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3064		ring = &adev->gfx.compute_ring[i];
3065		amdgpu_ring_test_helper(ring);
3066	}
3067
3068	return 0;
3069}
3070
3071static void gfx_v7_0_cp_enable(struct amdgpu_device *adev, bool enable)
3072{
3073	gfx_v7_0_cp_gfx_enable(adev, enable);
3074	gfx_v7_0_cp_compute_enable(adev, enable);
3075}
3076
3077static int gfx_v7_0_cp_load_microcode(struct amdgpu_device *adev)
3078{
3079	int r;
3080
3081	r = gfx_v7_0_cp_gfx_load_microcode(adev);
3082	if (r)
3083		return r;
3084	r = gfx_v7_0_cp_compute_load_microcode(adev);
3085	if (r)
3086		return r;
3087
3088	return 0;
3089}
3090
3091static void gfx_v7_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3092					       bool enable)
3093{
3094	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3095
3096	if (enable)
3097		tmp |= (CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK |
3098				CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK);
3099	else
3100		tmp &= ~(CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK |
3101				CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK);
3102	WREG32(mmCP_INT_CNTL_RING0, tmp);
3103}
3104
3105static int gfx_v7_0_cp_resume(struct amdgpu_device *adev)
3106{
3107	int r;
3108
3109	gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3110
3111	r = gfx_v7_0_cp_load_microcode(adev);
3112	if (r)
3113		return r;
3114
3115	r = gfx_v7_0_cp_gfx_resume(adev);
3116	if (r)
3117		return r;
3118	r = gfx_v7_0_cp_compute_resume(adev);
3119	if (r)
3120		return r;
3121
3122	gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3123
3124	return 0;
3125}
3126
3127/**
3128 * gfx_v7_0_ring_emit_pipeline_sync - cik vm flush using the CP
3129 *
3130 * @ring: the ring to emit the commands to
3131 *
3132 * Sync the command pipeline with the PFP. E.g. wait for everything
3133 * to be completed.
3134 */
3135static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3136{
3137	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3138	uint32_t seq = ring->fence_drv.sync_seq;
3139	uint64_t addr = ring->fence_drv.gpu_addr;
3140
3141	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3142	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
3143				 WAIT_REG_MEM_FUNCTION(3) | /* equal */
3144				 WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
3145	amdgpu_ring_write(ring, addr & 0xfffffffc);
3146	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3147	amdgpu_ring_write(ring, seq);
3148	amdgpu_ring_write(ring, 0xffffffff);
3149	amdgpu_ring_write(ring, 4); /* poll interval */
3150
3151	if (usepfp) {
3152		/* sync CE with ME to prevent CE fetch CEIB before context switch done */
3153		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3154		amdgpu_ring_write(ring, 0);
3155		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3156		amdgpu_ring_write(ring, 0);
3157	}
3158}
3159
3160/*
3161 * vm
3162 * VMID 0 is the physical GPU addresses as used by the kernel.
3163 * VMIDs 1-15 are used for userspace clients and are handled
3164 * by the amdgpu vm/hsa code.
3165 */
3166/**
3167 * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP
3168 *
3169 * @ring: amdgpu_ring pointer
3170 * @vmid: vmid number to use
3171 * @pd_addr: address
3172 *
3173 * Update the page table base and flush the VM TLB
3174 * using the CP (CIK).
3175 */
3176static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3177					unsigned vmid, uint64_t pd_addr)
3178{
3179	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3180
3181	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3182
3183	/* wait for the invalidate to complete */
3184	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3185	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
3186				 WAIT_REG_MEM_FUNCTION(0) |  /* always */
3187				 WAIT_REG_MEM_ENGINE(0))); /* me */
3188	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
3189	amdgpu_ring_write(ring, 0);
3190	amdgpu_ring_write(ring, 0); /* ref */
3191	amdgpu_ring_write(ring, 0); /* mask */
3192	amdgpu_ring_write(ring, 0x20); /* poll interval */
3193
3194	/* compute doesn't have PFP */
3195	if (usepfp) {
3196		/* sync PFP to ME, otherwise we might get invalid PFP reads */
3197		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3198		amdgpu_ring_write(ring, 0x0);
3199
3200		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
3201		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3202		amdgpu_ring_write(ring, 0);
3203		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3204		amdgpu_ring_write(ring, 0);
3205	}
3206}
3207
3208static void gfx_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,
3209				    uint32_t reg, uint32_t val)
3210{
3211	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3212
3213	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3214	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
3215				 WRITE_DATA_DST_SEL(0)));
3216	amdgpu_ring_write(ring, reg);
3217	amdgpu_ring_write(ring, 0);
3218	amdgpu_ring_write(ring, val);
3219}
3220
3221/*
3222 * RLC
3223 * The RLC is a multi-purpose microengine that handles a
3224 * variety of functions.
3225 */
3226static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
3227{
3228	const u32 *src_ptr;
3229	u32 dws;
3230	const struct cs_section_def *cs_data;
3231	int r;
3232
3233	/* allocate rlc buffers */
3234	if (adev->flags & AMD_IS_APU) {
3235		if (adev->asic_type == CHIP_KAVERI) {
3236			adev->gfx.rlc.reg_list = spectre_rlc_save_restore_register_list;
3237			adev->gfx.rlc.reg_list_size =
3238				(u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
3239		} else {
3240			adev->gfx.rlc.reg_list = kalindi_rlc_save_restore_register_list;
3241			adev->gfx.rlc.reg_list_size =
3242				(u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
3243		}
3244	}
3245	adev->gfx.rlc.cs_data = ci_cs_data;
3246	adev->gfx.rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
3247	adev->gfx.rlc.cp_table_size += 64 * 1024; /* GDS */
3248
3249	src_ptr = adev->gfx.rlc.reg_list;
3250	dws = adev->gfx.rlc.reg_list_size;
3251	dws += (5 * 16) + 48 + 48 + 64;
3252
3253	cs_data = adev->gfx.rlc.cs_data;
3254
3255	if (src_ptr) {
3256		/* init save restore block */
3257		r = amdgpu_gfx_rlc_init_sr(adev, dws);
3258		if (r)
3259			return r;
3260	}
3261
3262	if (cs_data) {
3263		/* init clear state block */
3264		r = amdgpu_gfx_rlc_init_csb(adev);
3265		if (r)
3266			return r;
3267	}
3268
3269	if (adev->gfx.rlc.cp_table_size) {
3270		r = amdgpu_gfx_rlc_init_cpt(adev);
3271		if (r)
3272			return r;
3273	}
3274
3275	/* init spm vmid with 0xf */
3276	if (adev->gfx.rlc.funcs->update_spm_vmid)
3277		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
3278
3279	return 0;
3280}
3281
3282static void gfx_v7_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
3283{
3284	u32 tmp;
3285
3286	tmp = RREG32(mmRLC_LB_CNTL);
3287	if (enable)
3288		tmp |= RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK;
3289	else
3290		tmp &= ~RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK;
3291	WREG32(mmRLC_LB_CNTL, tmp);
3292}
3293
3294static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3295{
3296	u32 i, j, k;
3297	u32 mask;
3298
3299	mutex_lock(&adev->grbm_idx_mutex);
3300	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3301		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3302			gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff, 0);
3303			for (k = 0; k < adev->usec_timeout; k++) {
3304				if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3305					break;
3306				udelay(1);
3307			}
3308		}
3309	}
3310	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3311	mutex_unlock(&adev->grbm_idx_mutex);
3312
3313	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3314		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3315		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3316		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3317	for (k = 0; k < adev->usec_timeout; k++) {
3318		if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3319			break;
3320		udelay(1);
3321	}
3322}
3323
3324static void gfx_v7_0_update_rlc(struct amdgpu_device *adev, u32 rlc)
3325{
3326	u32 tmp;
3327
3328	tmp = RREG32(mmRLC_CNTL);
3329	if (tmp != rlc)
3330		WREG32(mmRLC_CNTL, rlc);
3331}
3332
3333static u32 gfx_v7_0_halt_rlc(struct amdgpu_device *adev)
3334{
3335	u32 data, orig;
3336
3337	orig = data = RREG32(mmRLC_CNTL);
3338
3339	if (data & RLC_CNTL__RLC_ENABLE_F32_MASK) {
3340		u32 i;
3341
3342		data &= ~RLC_CNTL__RLC_ENABLE_F32_MASK;
3343		WREG32(mmRLC_CNTL, data);
3344
3345		for (i = 0; i < adev->usec_timeout; i++) {
3346			if ((RREG32(mmRLC_GPM_STAT) & RLC_GPM_STAT__RLC_BUSY_MASK) == 0)
3347				break;
3348			udelay(1);
3349		}
3350
3351		gfx_v7_0_wait_for_rlc_serdes(adev);
3352	}
3353
3354	return orig;
3355}
3356
3357static bool gfx_v7_0_is_rlc_enabled(struct amdgpu_device *adev)
3358{
3359	return true;
3360}
3361
3362static void gfx_v7_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
3363{
3364	u32 tmp, i, mask;
3365
3366	tmp = 0x1 | (1 << 1);
3367	WREG32(mmRLC_GPR_REG2, tmp);
3368
3369	mask = RLC_GPM_STAT__GFX_POWER_STATUS_MASK |
3370		RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK;
3371	for (i = 0; i < adev->usec_timeout; i++) {
3372		if ((RREG32(mmRLC_GPM_STAT) & mask) == mask)
3373			break;
3374		udelay(1);
3375	}
3376
3377	for (i = 0; i < adev->usec_timeout; i++) {
3378		if ((RREG32(mmRLC_GPR_REG2) & 0x1) == 0)
3379			break;
3380		udelay(1);
3381	}
3382}
3383
3384static void gfx_v7_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
3385{
3386	u32 tmp;
3387
3388	tmp = 0x1 | (0 << 1);
3389	WREG32(mmRLC_GPR_REG2, tmp);
3390}
3391
3392/**
3393 * gfx_v7_0_rlc_stop - stop the RLC ME
3394 *
3395 * @adev: amdgpu_device pointer
3396 *
3397 * Halt the RLC ME (MicroEngine) (CIK).
3398 */
3399static void gfx_v7_0_rlc_stop(struct amdgpu_device *adev)
3400{
3401	WREG32(mmRLC_CNTL, 0);
3402
3403	gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3404
3405	gfx_v7_0_wait_for_rlc_serdes(adev);
3406}
3407
3408/**
3409 * gfx_v7_0_rlc_start - start the RLC ME
3410 *
3411 * @adev: amdgpu_device pointer
3412 *
3413 * Unhalt the RLC ME (MicroEngine) (CIK).
3414 */
3415static void gfx_v7_0_rlc_start(struct amdgpu_device *adev)
3416{
3417	WREG32(mmRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
3418
3419	gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3420
3421	udelay(50);
3422}
3423
3424static void gfx_v7_0_rlc_reset(struct amdgpu_device *adev)
3425{
3426	u32 tmp = RREG32(mmGRBM_SOFT_RESET);
3427
3428	tmp |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
3429	WREG32(mmGRBM_SOFT_RESET, tmp);
3430	udelay(50);
3431	tmp &= ~GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
3432	WREG32(mmGRBM_SOFT_RESET, tmp);
3433	udelay(50);
3434}
3435
3436/**
3437 * gfx_v7_0_rlc_resume - setup the RLC hw
3438 *
3439 * @adev: amdgpu_device pointer
3440 *
3441 * Initialize the RLC registers, load the ucode,
3442 * and start the RLC (CIK).
3443 * Returns 0 for success, -EINVAL if the ucode is not available.
3444 */
3445static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev)
3446{
3447	const struct rlc_firmware_header_v1_0 *hdr;
3448	const __le32 *fw_data;
3449	unsigned i, fw_size;
3450	u32 tmp;
3451
3452	if (!adev->gfx.rlc_fw)
3453		return -EINVAL;
3454
3455	hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
3456	amdgpu_ucode_print_rlc_hdr(&hdr->header);
3457	adev->gfx.rlc_fw_version = le32_to_cpu(hdr->header.ucode_version);
3458	adev->gfx.rlc_feature_version = le32_to_cpu(
3459					hdr->ucode_feature_version);
3460
3461	adev->gfx.rlc.funcs->stop(adev);
3462
3463	/* disable CG */
3464	tmp = RREG32(mmRLC_CGCG_CGLS_CTRL) & 0xfffffffc;
3465	WREG32(mmRLC_CGCG_CGLS_CTRL, tmp);
3466
3467	adev->gfx.rlc.funcs->reset(adev);
3468
3469	gfx_v7_0_init_pg(adev);
3470
3471	WREG32(mmRLC_LB_CNTR_INIT, 0);
3472	WREG32(mmRLC_LB_CNTR_MAX, 0x00008000);
3473
3474	mutex_lock(&adev->grbm_idx_mutex);
3475	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3476	WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff);
3477	WREG32(mmRLC_LB_PARAMS, 0x00600408);
3478	WREG32(mmRLC_LB_CNTL, 0x80000004);
3479	mutex_unlock(&adev->grbm_idx_mutex);
3480
3481	WREG32(mmRLC_MC_CNTL, 0);
3482	WREG32(mmRLC_UCODE_CNTL, 0);
3483
3484	fw_data = (const __le32 *)
3485		(adev->gfx.rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3486	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3487	WREG32(mmRLC_GPM_UCODE_ADDR, 0);
3488	for (i = 0; i < fw_size; i++)
3489		WREG32(mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3490	WREG32(mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3491
3492	/* XXX - find out what chips support lbpw */
3493	gfx_v7_0_enable_lbpw(adev, false);
3494
3495	if (adev->asic_type == CHIP_BONAIRE)
3496		WREG32(mmRLC_DRIVER_CPDMA_STATUS, 0);
3497
3498	adev->gfx.rlc.funcs->start(adev);
3499
3500	return 0;
3501}
3502
3503static void gfx_v7_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
3504{
3505	u32 data;
3506
3507	amdgpu_gfx_off_ctrl(adev, false);
3508
3509	data = RREG32(mmRLC_SPM_VMID);
3510
3511	data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK;
3512	data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT;
3513
3514	WREG32(mmRLC_SPM_VMID, data);
3515
3516	amdgpu_gfx_off_ctrl(adev, true);
3517}
3518
3519static void gfx_v7_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
3520{
3521	u32 data, orig, tmp, tmp2;
3522
3523	orig = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
3524
3525	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
3526		gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3527
3528		tmp = gfx_v7_0_halt_rlc(adev);
3529
3530		mutex_lock(&adev->grbm_idx_mutex);
3531		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3532		WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3533		WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3534		tmp2 = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
3535			RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0_MASK |
3536			RLC_SERDES_WR_CTRL__CGLS_ENABLE_MASK;
3537		WREG32(mmRLC_SERDES_WR_CTRL, tmp2);
3538		mutex_unlock(&adev->grbm_idx_mutex);
3539
3540		gfx_v7_0_update_rlc(adev, tmp);
3541
3542		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3543		if (orig != data)
3544			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
3545
3546	} else {
3547		gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3548
3549		RREG32(mmCB_CGTT_SCLK_CTRL);
3550		RREG32(mmCB_CGTT_SCLK_CTRL);
3551		RREG32(mmCB_CGTT_SCLK_CTRL);
3552		RREG32(mmCB_CGTT_SCLK_CTRL);
3553
3554		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
3555		if (orig != data)
3556			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
3557
3558		gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3559	}
3560}
3561
3562static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
3563{
3564	u32 data, orig, tmp = 0;
3565
3566	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
3567		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
3568			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
3569				orig = data = RREG32(mmCP_MEM_SLP_CNTL);
3570				data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3571				if (orig != data)
3572					WREG32(mmCP_MEM_SLP_CNTL, data);
3573			}
3574		}
3575
3576		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
3577		data |= 0x00000001;
3578		data &= 0xfffffffd;
3579		if (orig != data)
3580			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
3581
3582		tmp = gfx_v7_0_halt_rlc(adev);
3583
3584		mutex_lock(&adev->grbm_idx_mutex);
3585		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3586		WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3587		WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3588		data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
3589			RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0_MASK;
3590		WREG32(mmRLC_SERDES_WR_CTRL, data);
3591		mutex_unlock(&adev->grbm_idx_mutex);
3592
3593		gfx_v7_0_update_rlc(adev, tmp);
3594
3595		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
3596			orig = data = RREG32(mmCGTS_SM_CTRL_REG);
3597			data &= ~CGTS_SM_CTRL_REG__SM_MODE_MASK;
3598			data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
3599			data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
3600			data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
3601			if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
3602			    (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
3603				data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
3604			data &= ~CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK;
3605			data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
3606			data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
3607			if (orig != data)
3608				WREG32(mmCGTS_SM_CTRL_REG, data);
3609		}
3610	} else {
3611		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
3612		data |= 0x00000003;
3613		if (orig != data)
3614			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
3615
3616		data = RREG32(mmRLC_MEM_SLP_CNTL);
3617		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
3618			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
3619			WREG32(mmRLC_MEM_SLP_CNTL, data);
3620		}
3621
3622		data = RREG32(mmCP_MEM_SLP_CNTL);
3623		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
3624			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3625			WREG32(mmCP_MEM_SLP_CNTL, data);
3626		}
3627
3628		orig = data = RREG32(mmCGTS_SM_CTRL_REG);
3629		data |= CGTS_SM_CTRL_REG__OVERRIDE_MASK | CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
3630		if (orig != data)
3631			WREG32(mmCGTS_SM_CTRL_REG, data);
3632
3633		tmp = gfx_v7_0_halt_rlc(adev);
3634
3635		mutex_lock(&adev->grbm_idx_mutex);
3636		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3637		WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3638		WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3639		data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK;
3640		WREG32(mmRLC_SERDES_WR_CTRL, data);
3641		mutex_unlock(&adev->grbm_idx_mutex);
3642
3643		gfx_v7_0_update_rlc(adev, tmp);
3644	}
3645}
3646
3647static void gfx_v7_0_update_cg(struct amdgpu_device *adev,
3648			       bool enable)
3649{
3650	gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3651	/* order matters! */
3652	if (enable) {
3653		gfx_v7_0_enable_mgcg(adev, true);
3654		gfx_v7_0_enable_cgcg(adev, true);
3655	} else {
3656		gfx_v7_0_enable_cgcg(adev, false);
3657		gfx_v7_0_enable_mgcg(adev, false);
3658	}
3659	gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3660}
3661
3662static void gfx_v7_0_enable_sclk_slowdown_on_pu(struct amdgpu_device *adev,
3663						bool enable)
3664{
3665	u32 data, orig;
3666
3667	orig = data = RREG32(mmRLC_PG_CNTL);
3668	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS))
3669		data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
3670	else
3671		data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
3672	if (orig != data)
3673		WREG32(mmRLC_PG_CNTL, data);
3674}
3675
3676static void gfx_v7_0_enable_sclk_slowdown_on_pd(struct amdgpu_device *adev,
3677						bool enable)
3678{
3679	u32 data, orig;
3680
3681	orig = data = RREG32(mmRLC_PG_CNTL);
3682	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS))
3683		data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
3684	else
3685		data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
3686	if (orig != data)
3687		WREG32(mmRLC_PG_CNTL, data);
3688}
3689
3690static void gfx_v7_0_enable_cp_pg(struct amdgpu_device *adev, bool enable)
3691{
3692	u32 data, orig;
3693
3694	orig = data = RREG32(mmRLC_PG_CNTL);
3695	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP))
3696		data &= ~0x8000;
3697	else
3698		data |= 0x8000;
3699	if (orig != data)
3700		WREG32(mmRLC_PG_CNTL, data);
3701}
3702
3703static void gfx_v7_0_enable_gds_pg(struct amdgpu_device *adev, bool enable)
3704{
3705	u32 data, orig;
3706
3707	orig = data = RREG32(mmRLC_PG_CNTL);
3708	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GDS))
3709		data &= ~0x2000;
3710	else
3711		data |= 0x2000;
3712	if (orig != data)
3713		WREG32(mmRLC_PG_CNTL, data);
3714}
3715
3716static int gfx_v7_0_cp_pg_table_num(struct amdgpu_device *adev)
3717{
3718	if (adev->asic_type == CHIP_KAVERI)
3719		return 5;
3720	else
3721		return 4;
3722}
3723
3724static void gfx_v7_0_enable_gfx_cgpg(struct amdgpu_device *adev,
3725				     bool enable)
3726{
3727	u32 data, orig;
3728
3729	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
3730		orig = data = RREG32(mmRLC_PG_CNTL);
3731		data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
3732		if (orig != data)
3733			WREG32(mmRLC_PG_CNTL, data);
3734
3735		orig = data = RREG32(mmRLC_AUTO_PG_CTRL);
3736		data |= RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK;
3737		if (orig != data)
3738			WREG32(mmRLC_AUTO_PG_CTRL, data);
3739	} else {
3740		orig = data = RREG32(mmRLC_PG_CNTL);
3741		data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
3742		if (orig != data)
3743			WREG32(mmRLC_PG_CNTL, data);
3744
3745		orig = data = RREG32(mmRLC_AUTO_PG_CTRL);
3746		data &= ~RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK;
3747		if (orig != data)
3748			WREG32(mmRLC_AUTO_PG_CTRL, data);
3749
3750		data = RREG32(mmDB_RENDER_CONTROL);
3751	}
3752}
3753
3754static void gfx_v7_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
3755						 u32 bitmap)
3756{
3757	u32 data;
3758
3759	if (!bitmap)
3760		return;
3761
3762	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
3763	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3764
3765	WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
3766}
3767
3768static u32 gfx_v7_0_get_cu_active_bitmap(struct amdgpu_device *adev)
3769{
3770	u32 data, mask;
3771
3772	data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG);
3773	data |= RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
3774
3775	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3776	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
3777
3778	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
3779
3780	return (~data) & mask;
3781}
3782
3783static void gfx_v7_0_init_ao_cu_mask(struct amdgpu_device *adev)
3784{
3785	u32 tmp;
3786
3787	WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
3788
3789	tmp = RREG32(mmRLC_MAX_PG_CU);
3790	tmp &= ~RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK;
3791	tmp |= (adev->gfx.cu_info.number << RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT);
3792	WREG32(mmRLC_MAX_PG_CU, tmp);
3793}
3794
3795static void gfx_v7_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
3796					    bool enable)
3797{
3798	u32 data, orig;
3799
3800	orig = data = RREG32(mmRLC_PG_CNTL);
3801	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG))
3802		data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
3803	else
3804		data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
3805	if (orig != data)
3806		WREG32(mmRLC_PG_CNTL, data);
3807}
3808
3809static void gfx_v7_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
3810					     bool enable)
3811{
3812	u32 data, orig;
3813
3814	orig = data = RREG32(mmRLC_PG_CNTL);
3815	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG))
3816		data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
3817	else
3818		data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
3819	if (orig != data)
3820		WREG32(mmRLC_PG_CNTL, data);
3821}
3822
3823#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
3824#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
3825
3826static void gfx_v7_0_init_gfx_cgpg(struct amdgpu_device *adev)
3827{
3828	u32 data, orig;
3829	u32 i;
3830
3831	if (adev->gfx.rlc.cs_data) {
3832		WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
3833		WREG32(mmRLC_GPM_SCRATCH_DATA, upper_32_bits(adev->gfx.rlc.clear_state_gpu_addr));
3834		WREG32(mmRLC_GPM_SCRATCH_DATA, lower_32_bits(adev->gfx.rlc.clear_state_gpu_addr));
3835		WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.clear_state_size);
3836	} else {
3837		WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
3838		for (i = 0; i < 3; i++)
3839			WREG32(mmRLC_GPM_SCRATCH_DATA, 0);
3840	}
3841	if (adev->gfx.rlc.reg_list) {
3842		WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
3843		for (i = 0; i < adev->gfx.rlc.reg_list_size; i++)
3844			WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.reg_list[i]);
3845	}
3846
3847	orig = data = RREG32(mmRLC_PG_CNTL);
3848	data |= RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK;
3849	if (orig != data)
3850		WREG32(mmRLC_PG_CNTL, data);
3851
3852	WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
3853	WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
3854
3855	data = RREG32(mmCP_RB_WPTR_POLL_CNTL);
3856	data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
3857	data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3858	WREG32(mmCP_RB_WPTR_POLL_CNTL, data);
3859
3860	data = 0x10101010;
3861	WREG32(mmRLC_PG_DELAY, data);
3862
3863	data = RREG32(mmRLC_PG_DELAY_2);
3864	data &= ~0xff;
3865	data |= 0x3;
3866	WREG32(mmRLC_PG_DELAY_2, data);
3867
3868	data = RREG32(mmRLC_AUTO_PG_CTRL);
3869	data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
3870	data |= (0x700 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
3871	WREG32(mmRLC_AUTO_PG_CTRL, data);
3872
3873}
3874
3875static void gfx_v7_0_update_gfx_pg(struct amdgpu_device *adev, bool enable)
3876{
3877	gfx_v7_0_enable_gfx_cgpg(adev, enable);
3878	gfx_v7_0_enable_gfx_static_mgpg(adev, enable);
3879	gfx_v7_0_enable_gfx_dynamic_mgpg(adev, enable);
3880}
3881
3882static u32 gfx_v7_0_get_csb_size(struct amdgpu_device *adev)
3883{
3884	u32 count = 0;
3885	const struct cs_section_def *sect = NULL;
3886	const struct cs_extent_def *ext = NULL;
3887
3888	if (adev->gfx.rlc.cs_data == NULL)
3889		return 0;
3890
3891	/* begin clear state */
3892	count += 2;
3893	/* context control state */
3894	count += 3;
3895
3896	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
3897		for (ext = sect->section; ext->extent != NULL; ++ext) {
3898			if (sect->id == SECT_CONTEXT)
3899				count += 2 + ext->reg_count;
3900			else
3901				return 0;
3902		}
3903	}
3904	/* pa_sc_raster_config/pa_sc_raster_config1 */
3905	count += 4;
3906	/* end clear state */
3907	count += 2;
3908	/* clear state */
3909	count += 2;
3910
3911	return count;
3912}
3913
3914static void gfx_v7_0_get_csb_buffer(struct amdgpu_device *adev,
3915				    volatile u32 *buffer)
3916{
3917	u32 count = 0, i;
3918	const struct cs_section_def *sect = NULL;
3919	const struct cs_extent_def *ext = NULL;
3920
3921	if (adev->gfx.rlc.cs_data == NULL)
3922		return;
3923	if (buffer == NULL)
3924		return;
3925
3926	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3927	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3928
3929	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3930	buffer[count++] = cpu_to_le32(0x80000000);
3931	buffer[count++] = cpu_to_le32(0x80000000);
3932
3933	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
3934		for (ext = sect->section; ext->extent != NULL; ++ext) {
3935			if (sect->id == SECT_CONTEXT) {
3936				buffer[count++] =
3937					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
3938				buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3939				for (i = 0; i < ext->reg_count; i++)
3940					buffer[count++] = cpu_to_le32(ext->extent[i]);
3941			} else {
3942				return;
3943			}
3944		}
3945	}
3946
3947	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3948	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
3949	switch (adev->asic_type) {
3950	case CHIP_BONAIRE:
3951		buffer[count++] = cpu_to_le32(0x16000012);
3952		buffer[count++] = cpu_to_le32(0x00000000);
3953		break;
3954	case CHIP_KAVERI:
3955		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
3956		buffer[count++] = cpu_to_le32(0x00000000);
3957		break;
3958	case CHIP_KABINI:
3959	case CHIP_MULLINS:
3960		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
3961		buffer[count++] = cpu_to_le32(0x00000000);
3962		break;
3963	case CHIP_HAWAII:
3964		buffer[count++] = cpu_to_le32(0x3a00161a);
3965		buffer[count++] = cpu_to_le32(0x0000002e);
3966		break;
3967	default:
3968		buffer[count++] = cpu_to_le32(0x00000000);
3969		buffer[count++] = cpu_to_le32(0x00000000);
3970		break;
3971	}
3972
3973	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3974	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
3975
3976	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
3977	buffer[count++] = cpu_to_le32(0);
3978}
3979
3980static void gfx_v7_0_init_pg(struct amdgpu_device *adev)
3981{
3982	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3983			      AMD_PG_SUPPORT_GFX_SMG |
3984			      AMD_PG_SUPPORT_GFX_DMG |
3985			      AMD_PG_SUPPORT_CP |
3986			      AMD_PG_SUPPORT_GDS |
3987			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
3988		gfx_v7_0_enable_sclk_slowdown_on_pu(adev, true);
3989		gfx_v7_0_enable_sclk_slowdown_on_pd(adev, true);
3990		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
3991			gfx_v7_0_init_gfx_cgpg(adev);
3992			gfx_v7_0_enable_cp_pg(adev, true);
3993			gfx_v7_0_enable_gds_pg(adev, true);
3994		}
3995		gfx_v7_0_init_ao_cu_mask(adev);
3996		gfx_v7_0_update_gfx_pg(adev, true);
3997	}
3998}
3999
4000static void gfx_v7_0_fini_pg(struct amdgpu_device *adev)
4001{
4002	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
4003			      AMD_PG_SUPPORT_GFX_SMG |
4004			      AMD_PG_SUPPORT_GFX_DMG |
4005			      AMD_PG_SUPPORT_CP |
4006			      AMD_PG_SUPPORT_GDS |
4007			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
4008		gfx_v7_0_update_gfx_pg(adev, false);
4009		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
4010			gfx_v7_0_enable_cp_pg(adev, false);
4011			gfx_v7_0_enable_gds_pg(adev, false);
4012		}
4013	}
4014}
4015
4016/**
4017 * gfx_v7_0_get_gpu_clock_counter - return GPU clock counter snapshot
4018 *
4019 * @adev: amdgpu_device pointer
4020 *
4021 * Fetches a GPU clock counter snapshot (SI).
4022 * Returns the 64 bit clock counter snapshot.
4023 */
4024static uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4025{
4026	uint64_t clock;
4027
4028	mutex_lock(&adev->gfx.gpu_clock_mutex);
4029	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4030	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
4031		((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4032	mutex_unlock(&adev->gfx.gpu_clock_mutex);
4033	return clock;
4034}
4035
4036static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4037					  uint32_t vmid,
4038					  uint32_t gds_base, uint32_t gds_size,
4039					  uint32_t gws_base, uint32_t gws_size,
4040					  uint32_t oa_base, uint32_t oa_size)
4041{
4042	/* GDS Base */
4043	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4044	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4045				WRITE_DATA_DST_SEL(0)));
4046	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
4047	amdgpu_ring_write(ring, 0);
4048	amdgpu_ring_write(ring, gds_base);
4049
4050	/* GDS Size */
4051	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4052	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4053				WRITE_DATA_DST_SEL(0)));
4054	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
4055	amdgpu_ring_write(ring, 0);
4056	amdgpu_ring_write(ring, gds_size);
4057
4058	/* GWS */
4059	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4060	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4061				WRITE_DATA_DST_SEL(0)));
4062	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
4063	amdgpu_ring_write(ring, 0);
4064	amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4065
4066	/* OA */
4067	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4068	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4069				WRITE_DATA_DST_SEL(0)));
4070	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
4071	amdgpu_ring_write(ring, 0);
4072	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
4073}
4074
4075static void gfx_v7_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
4076{
4077	struct amdgpu_device *adev = ring->adev;
4078	uint32_t value = 0;
4079
4080	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
4081	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
4082	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
4083	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
4084	WREG32(mmSQ_CMD, value);
4085}
4086
4087static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
4088{
4089	WREG32(mmSQ_IND_INDEX,
4090		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
4091		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
4092		(address << SQ_IND_INDEX__INDEX__SHIFT) |
4093		(SQ_IND_INDEX__FORCE_READ_MASK));
4094	return RREG32(mmSQ_IND_DATA);
4095}
4096
4097static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
4098			   uint32_t wave, uint32_t thread,
4099			   uint32_t regno, uint32_t num, uint32_t *out)
4100{
4101	WREG32(mmSQ_IND_INDEX,
4102		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
4103		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
4104		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
4105		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
4106		(SQ_IND_INDEX__FORCE_READ_MASK) |
4107		(SQ_IND_INDEX__AUTO_INCR_MASK));
4108	while (num--)
4109		*(out++) = RREG32(mmSQ_IND_DATA);
4110}
4111
4112static void gfx_v7_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
4113{
4114	/* type 0 wave data */
4115	dst[(*no_fields)++] = 0;
4116	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
4117	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
4118	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
4119	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
4120	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
4121	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
4122	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
4123	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
4124	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
4125	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
4126	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
4127	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
4128	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
4129	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
4130	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
4131	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
4132	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
4133	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
4134	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
4135}
4136
4137static void gfx_v7_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
4138				     uint32_t wave, uint32_t start,
4139				     uint32_t size, uint32_t *dst)
4140{
4141	wave_read_regs(
4142		adev, simd, wave, 0,
4143		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
4144}
4145
4146static void gfx_v7_0_select_me_pipe_q(struct amdgpu_device *adev,
4147				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
4148{
4149	cik_srbm_select(adev, me, pipe, q, vm);
4150}
4151
4152static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
4153	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
4154	.select_se_sh = &gfx_v7_0_select_se_sh,
4155	.read_wave_data = &gfx_v7_0_read_wave_data,
4156	.read_wave_sgprs = &gfx_v7_0_read_wave_sgprs,
4157	.select_me_pipe_q = &gfx_v7_0_select_me_pipe_q
4158};
4159
4160static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = {
4161	.is_rlc_enabled = gfx_v7_0_is_rlc_enabled,
4162	.set_safe_mode = gfx_v7_0_set_safe_mode,
4163	.unset_safe_mode = gfx_v7_0_unset_safe_mode,
4164	.init = gfx_v7_0_rlc_init,
4165	.get_csb_size = gfx_v7_0_get_csb_size,
4166	.get_csb_buffer = gfx_v7_0_get_csb_buffer,
4167	.get_cp_table_num = gfx_v7_0_cp_pg_table_num,
4168	.resume = gfx_v7_0_rlc_resume,
4169	.stop = gfx_v7_0_rlc_stop,
4170	.reset = gfx_v7_0_rlc_reset,
4171	.start = gfx_v7_0_rlc_start,
4172	.update_spm_vmid = gfx_v7_0_update_spm_vmid
4173};
4174
4175static int gfx_v7_0_early_init(void *handle)
4176{
4177	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4178
4179	adev->gfx.xcc_mask = 1;
4180	adev->gfx.num_gfx_rings = GFX7_NUM_GFX_RINGS;
4181	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4182					  AMDGPU_MAX_COMPUTE_RINGS);
4183	adev->gfx.funcs = &gfx_v7_0_gfx_funcs;
4184	adev->gfx.rlc.funcs = &gfx_v7_0_rlc_funcs;
4185	gfx_v7_0_set_ring_funcs(adev);
4186	gfx_v7_0_set_irq_funcs(adev);
4187	gfx_v7_0_set_gds_init(adev);
4188
4189	return 0;
4190}
4191
4192static int gfx_v7_0_late_init(void *handle)
4193{
4194	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4195	int r;
4196
4197	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4198	if (r)
4199		return r;
4200
4201	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4202	if (r)
4203		return r;
4204
4205	return 0;
4206}
4207
4208static void gfx_v7_0_gpu_early_init(struct amdgpu_device *adev)
4209{
4210	u32 gb_addr_config;
4211	u32 mc_arb_ramcfg;
4212	u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
4213	u32 tmp;
4214
4215	switch (adev->asic_type) {
4216	case CHIP_BONAIRE:
4217		adev->gfx.config.max_shader_engines = 2;
4218		adev->gfx.config.max_tile_pipes = 4;
4219		adev->gfx.config.max_cu_per_sh = 7;
4220		adev->gfx.config.max_sh_per_se = 1;
4221		adev->gfx.config.max_backends_per_se = 2;
4222		adev->gfx.config.max_texture_channel_caches = 4;
4223		adev->gfx.config.max_gprs = 256;
4224		adev->gfx.config.max_gs_threads = 32;
4225		adev->gfx.config.max_hw_contexts = 8;
4226
4227		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4228		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4229		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4230		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4231		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4232		break;
4233	case CHIP_HAWAII:
4234		adev->gfx.config.max_shader_engines = 4;
4235		adev->gfx.config.max_tile_pipes = 16;
4236		adev->gfx.config.max_cu_per_sh = 11;
4237		adev->gfx.config.max_sh_per_se = 1;
4238		adev->gfx.config.max_backends_per_se = 4;
4239		adev->gfx.config.max_texture_channel_caches = 16;
4240		adev->gfx.config.max_gprs = 256;
4241		adev->gfx.config.max_gs_threads = 32;
4242		adev->gfx.config.max_hw_contexts = 8;
4243
4244		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4245		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4246		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4247		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4248		gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
4249		break;
4250	case CHIP_KAVERI:
4251		adev->gfx.config.max_shader_engines = 1;
4252		adev->gfx.config.max_tile_pipes = 4;
4253		adev->gfx.config.max_cu_per_sh = 8;
4254		adev->gfx.config.max_backends_per_se = 2;
4255		adev->gfx.config.max_sh_per_se = 1;
4256		adev->gfx.config.max_texture_channel_caches = 4;
4257		adev->gfx.config.max_gprs = 256;
4258		adev->gfx.config.max_gs_threads = 16;
4259		adev->gfx.config.max_hw_contexts = 8;
4260
4261		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4262		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4263		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4264		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4265		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4266		break;
4267	case CHIP_KABINI:
4268	case CHIP_MULLINS:
4269	default:
4270		adev->gfx.config.max_shader_engines = 1;
4271		adev->gfx.config.max_tile_pipes = 2;
4272		adev->gfx.config.max_cu_per_sh = 2;
4273		adev->gfx.config.max_sh_per_se = 1;
4274		adev->gfx.config.max_backends_per_se = 1;
4275		adev->gfx.config.max_texture_channel_caches = 2;
4276		adev->gfx.config.max_gprs = 256;
4277		adev->gfx.config.max_gs_threads = 16;
4278		adev->gfx.config.max_hw_contexts = 8;
4279
4280		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4281		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4282		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4283		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4284		gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4285		break;
4286	}
4287
4288	adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
4289	mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
4290
4291	adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg,
4292				MC_ARB_RAMCFG, NOOFBANK);
4293	adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg,
4294				MC_ARB_RAMCFG, NOOFRANKS);
4295
4296	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
4297	adev->gfx.config.mem_max_burst_length_bytes = 256;
4298	if (adev->flags & AMD_IS_APU) {
4299		/* Get memory bank mapping mode. */
4300		tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
4301		dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
4302		dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
4303
4304		tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
4305		dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
4306		dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
4307
4308		/* Validate settings in case only one DIMM installed. */
4309		if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
4310			dimm00_addr_map = 0;
4311		if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
4312			dimm01_addr_map = 0;
4313		if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
4314			dimm10_addr_map = 0;
4315		if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
4316			dimm11_addr_map = 0;
4317
4318		/* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
4319		/* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
4320		if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
4321			adev->gfx.config.mem_row_size_in_kb = 2;
4322		else
4323			adev->gfx.config.mem_row_size_in_kb = 1;
4324	} else {
4325		tmp = (mc_arb_ramcfg & MC_ARB_RAMCFG__NOOFCOLS_MASK) >> MC_ARB_RAMCFG__NOOFCOLS__SHIFT;
4326		adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
4327		if (adev->gfx.config.mem_row_size_in_kb > 4)
4328			adev->gfx.config.mem_row_size_in_kb = 4;
4329	}
4330	/* XXX use MC settings? */
4331	adev->gfx.config.shader_engine_tile_size = 32;
4332	adev->gfx.config.num_gpus = 1;
4333	adev->gfx.config.multi_gpu_tile_size = 64;
4334
4335	/* fix up row size */
4336	gb_addr_config &= ~GB_ADDR_CONFIG__ROW_SIZE_MASK;
4337	switch (adev->gfx.config.mem_row_size_in_kb) {
4338	case 1:
4339	default:
4340		gb_addr_config |= (0 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4341		break;
4342	case 2:
4343		gb_addr_config |= (1 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4344		break;
4345	case 4:
4346		gb_addr_config |= (2 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4347		break;
4348	}
4349	adev->gfx.config.gb_addr_config = gb_addr_config;
4350}
4351
4352static int gfx_v7_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
4353					int mec, int pipe, int queue)
4354{
4355	int r;
4356	unsigned irq_type;
4357	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
4358
4359	/* mec0 is me1 */
4360	ring->me = mec + 1;
4361	ring->pipe = pipe;
4362	ring->queue = queue;
4363
4364	ring->ring_obj = NULL;
4365	ring->use_doorbell = true;
4366	ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
4367	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
4368
4369	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4370		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
4371		+ ring->pipe;
4372
4373	/* type-2 packets are deprecated on MEC, use type-3 instead */
4374	r = amdgpu_ring_init(adev, ring, 1024,
4375			     &adev->gfx.eop_irq, irq_type,
4376			     AMDGPU_RING_PRIO_DEFAULT, NULL);
4377	if (r)
4378		return r;
4379
4380
4381	return 0;
4382}
4383
4384static int gfx_v7_0_sw_init(void *handle)
4385{
4386	struct amdgpu_ring *ring;
4387	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4388	int i, j, k, r, ring_id;
4389
4390	switch (adev->asic_type) {
4391	case CHIP_KAVERI:
4392		adev->gfx.mec.num_mec = 2;
4393		break;
4394	case CHIP_BONAIRE:
4395	case CHIP_HAWAII:
4396	case CHIP_KABINI:
4397	case CHIP_MULLINS:
4398	default:
4399		adev->gfx.mec.num_mec = 1;
4400		break;
4401	}
4402	adev->gfx.mec.num_pipe_per_mec = 4;
4403	adev->gfx.mec.num_queue_per_pipe = 8;
4404
4405	/* EOP Event */
4406	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 181, &adev->gfx.eop_irq);
4407	if (r)
4408		return r;
4409
4410	/* Privileged reg */
4411	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 184,
4412			      &adev->gfx.priv_reg_irq);
4413	if (r)
4414		return r;
4415
4416	/* Privileged inst */
4417	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 185,
4418			      &adev->gfx.priv_inst_irq);
4419	if (r)
4420		return r;
4421
4422	r = gfx_v7_0_init_microcode(adev);
4423	if (r) {
4424		DRM_ERROR("Failed to load gfx firmware!\n");
4425		return r;
4426	}
4427
4428	r = adev->gfx.rlc.funcs->init(adev);
4429	if (r) {
4430		DRM_ERROR("Failed to init rlc BOs!\n");
4431		return r;
4432	}
4433
4434	/* allocate mec buffers */
4435	r = gfx_v7_0_mec_init(adev);
4436	if (r) {
4437		DRM_ERROR("Failed to init MEC BOs!\n");
4438		return r;
4439	}
4440
4441	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4442		ring = &adev->gfx.gfx_ring[i];
4443		ring->ring_obj = NULL;
4444		sprintf(ring->name, "gfx");
4445		r = amdgpu_ring_init(adev, ring, 1024,
4446				     &adev->gfx.eop_irq,
4447				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
4448				     AMDGPU_RING_PRIO_DEFAULT, NULL);
4449		if (r)
4450			return r;
4451	}
4452
4453	/* set up the compute queues - allocate horizontally across pipes */
4454	ring_id = 0;
4455	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4456		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4457			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4458				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
4459								     k, j))
4460					continue;
4461
4462				r = gfx_v7_0_compute_ring_init(adev,
4463								ring_id,
4464								i, k, j);
4465				if (r)
4466					return r;
4467
4468				ring_id++;
4469			}
4470		}
4471	}
4472
4473	adev->gfx.ce_ram_size = 0x8000;
4474
4475	gfx_v7_0_gpu_early_init(adev);
4476
4477	return r;
4478}
4479
4480static int gfx_v7_0_sw_fini(void *handle)
4481{
4482	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4483	int i;
4484
4485	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4486		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
4487	for (i = 0; i < adev->gfx.num_compute_rings; i++)
4488		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
4489
4490	gfx_v7_0_cp_compute_fini(adev);
4491	amdgpu_gfx_rlc_fini(adev);
4492	gfx_v7_0_mec_fini(adev);
4493	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
4494				&adev->gfx.rlc.clear_state_gpu_addr,
4495				(void **)&adev->gfx.rlc.cs_ptr);
4496	if (adev->gfx.rlc.cp_table_size) {
4497		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
4498				&adev->gfx.rlc.cp_table_gpu_addr,
4499				(void **)&adev->gfx.rlc.cp_table_ptr);
4500	}
4501	gfx_v7_0_free_microcode(adev);
4502
4503	return 0;
4504}
4505
4506static int gfx_v7_0_hw_init(void *handle)
4507{
4508	int r;
4509	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4510
4511	gfx_v7_0_constants_init(adev);
4512
4513	/* init CSB */
4514	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
4515	/* init rlc */
4516	r = adev->gfx.rlc.funcs->resume(adev);
4517	if (r)
4518		return r;
4519
4520	r = gfx_v7_0_cp_resume(adev);
4521	if (r)
4522		return r;
4523
4524	return r;
4525}
4526
4527static int gfx_v7_0_hw_fini(void *handle)
4528{
4529	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4530
4531	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4532	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4533	gfx_v7_0_cp_enable(adev, false);
4534	adev->gfx.rlc.funcs->stop(adev);
4535	gfx_v7_0_fini_pg(adev);
4536
4537	return 0;
4538}
4539
4540static int gfx_v7_0_suspend(void *handle)
4541{
4542	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4543
4544	return gfx_v7_0_hw_fini(adev);
4545}
4546
4547static int gfx_v7_0_resume(void *handle)
4548{
4549	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4550
4551	return gfx_v7_0_hw_init(adev);
4552}
4553
4554static bool gfx_v7_0_is_idle(void *handle)
4555{
4556	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4557
4558	if (RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
4559		return false;
4560	else
4561		return true;
4562}
4563
4564static int gfx_v7_0_wait_for_idle(void *handle)
4565{
4566	unsigned i;
4567	u32 tmp;
4568	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4569
4570	for (i = 0; i < adev->usec_timeout; i++) {
4571		/* read MC_STATUS */
4572		tmp = RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK;
4573
4574		if (!tmp)
4575			return 0;
4576		udelay(1);
4577	}
4578	return -ETIMEDOUT;
4579}
4580
4581static int gfx_v7_0_soft_reset(void *handle)
4582{
4583	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4584	u32 tmp;
4585	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4586
4587	/* GRBM_STATUS */
4588	tmp = RREG32(mmGRBM_STATUS);
4589	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4590		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4591		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4592		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4593		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4594		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK))
4595		grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK |
4596			GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK;
4597
4598	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4599		grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK;
4600		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
4601	}
4602
4603	/* GRBM_STATUS2 */
4604	tmp = RREG32(mmGRBM_STATUS2);
4605	if (tmp & GRBM_STATUS2__RLC_BUSY_MASK)
4606		grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
4607
4608	/* SRBM_STATUS */
4609	tmp = RREG32(mmSRBM_STATUS);
4610	if (tmp & SRBM_STATUS__GRBM_RQ_PENDING_MASK)
4611		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
4612
4613	if (grbm_soft_reset || srbm_soft_reset) {
4614		/* disable CG/PG */
4615		gfx_v7_0_fini_pg(adev);
4616		gfx_v7_0_update_cg(adev, false);
4617
4618		/* stop the rlc */
4619		adev->gfx.rlc.funcs->stop(adev);
4620
4621		/* Disable GFX parsing/prefetching */
4622		WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
4623
4624		/* Disable MEC parsing/prefetching */
4625		WREG32(mmCP_MEC_CNTL, CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
4626
4627		if (grbm_soft_reset) {
4628			tmp = RREG32(mmGRBM_SOFT_RESET);
4629			tmp |= grbm_soft_reset;
4630			dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4631			WREG32(mmGRBM_SOFT_RESET, tmp);
4632			tmp = RREG32(mmGRBM_SOFT_RESET);
4633
4634			udelay(50);
4635
4636			tmp &= ~grbm_soft_reset;
4637			WREG32(mmGRBM_SOFT_RESET, tmp);
4638			tmp = RREG32(mmGRBM_SOFT_RESET);
4639		}
4640
4641		if (srbm_soft_reset) {
4642			tmp = RREG32(mmSRBM_SOFT_RESET);
4643			tmp |= srbm_soft_reset;
4644			dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4645			WREG32(mmSRBM_SOFT_RESET, tmp);
4646			tmp = RREG32(mmSRBM_SOFT_RESET);
4647
4648			udelay(50);
4649
4650			tmp &= ~srbm_soft_reset;
4651			WREG32(mmSRBM_SOFT_RESET, tmp);
4652			tmp = RREG32(mmSRBM_SOFT_RESET);
4653		}
4654		/* Wait a little for things to settle down */
4655		udelay(50);
4656	}
4657	return 0;
4658}
4659
4660static void gfx_v7_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4661						 enum amdgpu_interrupt_state state)
4662{
4663	u32 cp_int_cntl;
4664
4665	switch (state) {
4666	case AMDGPU_IRQ_STATE_DISABLE:
4667		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4668		cp_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4669		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4670		break;
4671	case AMDGPU_IRQ_STATE_ENABLE:
4672		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4673		cp_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4674		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4675		break;
4676	default:
4677		break;
4678	}
4679}
4680
4681static void gfx_v7_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4682						     int me, int pipe,
4683						     enum amdgpu_interrupt_state state)
4684{
4685	u32 mec_int_cntl, mec_int_cntl_reg;
4686
4687	/*
4688	 * amdgpu controls only the first MEC. That's why this function only
4689	 * handles the setting of interrupts for this specific MEC. All other
4690	 * pipes' interrupts are set by amdkfd.
4691	 */
4692
4693	if (me == 1) {
4694		switch (pipe) {
4695		case 0:
4696			mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
4697			break;
4698		case 1:
4699			mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
4700			break;
4701		case 2:
4702			mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
4703			break;
4704		case 3:
4705			mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
4706			break;
4707		default:
4708			DRM_DEBUG("invalid pipe %d\n", pipe);
4709			return;
4710		}
4711	} else {
4712		DRM_DEBUG("invalid me %d\n", me);
4713		return;
4714	}
4715
4716	switch (state) {
4717	case AMDGPU_IRQ_STATE_DISABLE:
4718		mec_int_cntl = RREG32(mec_int_cntl_reg);
4719		mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4720		WREG32(mec_int_cntl_reg, mec_int_cntl);
4721		break;
4722	case AMDGPU_IRQ_STATE_ENABLE:
4723		mec_int_cntl = RREG32(mec_int_cntl_reg);
4724		mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4725		WREG32(mec_int_cntl_reg, mec_int_cntl);
4726		break;
4727	default:
4728		break;
4729	}
4730}
4731
4732static int gfx_v7_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4733					     struct amdgpu_irq_src *src,
4734					     unsigned type,
4735					     enum amdgpu_interrupt_state state)
4736{
4737	u32 cp_int_cntl;
4738
4739	switch (state) {
4740	case AMDGPU_IRQ_STATE_DISABLE:
4741		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4742		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
4743		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4744		break;
4745	case AMDGPU_IRQ_STATE_ENABLE:
4746		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4747		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
4748		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4749		break;
4750	default:
4751		break;
4752	}
4753
4754	return 0;
4755}
4756
4757static int gfx_v7_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4758					      struct amdgpu_irq_src *src,
4759					      unsigned type,
4760					      enum amdgpu_interrupt_state state)
4761{
4762	u32 cp_int_cntl;
4763
4764	switch (state) {
4765	case AMDGPU_IRQ_STATE_DISABLE:
4766		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4767		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
4768		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4769		break;
4770	case AMDGPU_IRQ_STATE_ENABLE:
4771		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4772		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
4773		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4774		break;
4775	default:
4776		break;
4777	}
4778
4779	return 0;
4780}
4781
4782static int gfx_v7_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4783					    struct amdgpu_irq_src *src,
4784					    unsigned type,
4785					    enum amdgpu_interrupt_state state)
4786{
4787	switch (type) {
4788	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
4789		gfx_v7_0_set_gfx_eop_interrupt_state(adev, state);
4790		break;
4791	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4792		gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4793		break;
4794	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4795		gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4796		break;
4797	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4798		gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4799		break;
4800	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4801		gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4802		break;
4803	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
4804		gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
4805		break;
4806	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
4807		gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
4808		break;
4809	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
4810		gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
4811		break;
4812	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
4813		gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
4814		break;
4815	default:
4816		break;
4817	}
4818	return 0;
4819}
4820
4821static int gfx_v7_0_eop_irq(struct amdgpu_device *adev,
4822			    struct amdgpu_irq_src *source,
4823			    struct amdgpu_iv_entry *entry)
4824{
4825	u8 me_id, pipe_id;
4826	struct amdgpu_ring *ring;
4827	int i;
4828
4829	DRM_DEBUG("IH: CP EOP\n");
4830	me_id = (entry->ring_id & 0x0c) >> 2;
4831	pipe_id = (entry->ring_id & 0x03) >> 0;
4832	switch (me_id) {
4833	case 0:
4834		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4835		break;
4836	case 1:
4837	case 2:
4838		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4839			ring = &adev->gfx.compute_ring[i];
4840			if ((ring->me == me_id) && (ring->pipe == pipe_id))
4841				amdgpu_fence_process(ring);
4842		}
4843		break;
4844	}
4845	return 0;
4846}
4847
4848static void gfx_v7_0_fault(struct amdgpu_device *adev,
4849			   struct amdgpu_iv_entry *entry)
4850{
4851	struct amdgpu_ring *ring;
4852	u8 me_id, pipe_id;
4853	int i;
4854
4855	me_id = (entry->ring_id & 0x0c) >> 2;
4856	pipe_id = (entry->ring_id & 0x03) >> 0;
4857	switch (me_id) {
4858	case 0:
4859		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
4860		break;
4861	case 1:
4862	case 2:
4863		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4864			ring = &adev->gfx.compute_ring[i];
4865			if ((ring->me == me_id) && (ring->pipe == pipe_id))
4866				drm_sched_fault(&ring->sched);
4867		}
4868		break;
4869	}
4870}
4871
4872static int gfx_v7_0_priv_reg_irq(struct amdgpu_device *adev,
4873				 struct amdgpu_irq_src *source,
4874				 struct amdgpu_iv_entry *entry)
4875{
4876	DRM_ERROR("Illegal register access in command stream\n");
4877	gfx_v7_0_fault(adev, entry);
4878	return 0;
4879}
4880
4881static int gfx_v7_0_priv_inst_irq(struct amdgpu_device *adev,
4882				  struct amdgpu_irq_src *source,
4883				  struct amdgpu_iv_entry *entry)
4884{
4885	DRM_ERROR("Illegal instruction in command stream\n");
4886	// XXX soft reset the gfx block only
4887	gfx_v7_0_fault(adev, entry);
4888	return 0;
4889}
4890
4891static int gfx_v7_0_set_clockgating_state(void *handle,
4892					  enum amd_clockgating_state state)
4893{
4894	bool gate = false;
4895	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4896
4897	if (state == AMD_CG_STATE_GATE)
4898		gate = true;
4899
4900	gfx_v7_0_enable_gui_idle_interrupt(adev, false);
4901	/* order matters! */
4902	if (gate) {
4903		gfx_v7_0_enable_mgcg(adev, true);
4904		gfx_v7_0_enable_cgcg(adev, true);
4905	} else {
4906		gfx_v7_0_enable_cgcg(adev, false);
4907		gfx_v7_0_enable_mgcg(adev, false);
4908	}
4909	gfx_v7_0_enable_gui_idle_interrupt(adev, true);
4910
4911	return 0;
4912}
4913
4914static int gfx_v7_0_set_powergating_state(void *handle,
4915					  enum amd_powergating_state state)
4916{
4917	bool gate = false;
4918	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4919
4920	if (state == AMD_PG_STATE_GATE)
4921		gate = true;
4922
4923	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
4924			      AMD_PG_SUPPORT_GFX_SMG |
4925			      AMD_PG_SUPPORT_GFX_DMG |
4926			      AMD_PG_SUPPORT_CP |
4927			      AMD_PG_SUPPORT_GDS |
4928			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
4929		gfx_v7_0_update_gfx_pg(adev, gate);
4930		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
4931			gfx_v7_0_enable_cp_pg(adev, gate);
4932			gfx_v7_0_enable_gds_pg(adev, gate);
4933		}
4934	}
4935
4936	return 0;
4937}
4938
4939static void gfx_v7_0_emit_mem_sync(struct amdgpu_ring *ring)
4940{
4941	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
4942	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
4943			  PACKET3_TC_ACTION_ENA |
4944			  PACKET3_SH_KCACHE_ACTION_ENA |
4945			  PACKET3_SH_ICACHE_ACTION_ENA);  /* CP_COHER_CNTL */
4946	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
4947	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE */
4948	amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
4949}
4950
4951static void gfx_v7_0_emit_mem_sync_compute(struct amdgpu_ring *ring)
4952{
4953	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
4954	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
4955			  PACKET3_TC_ACTION_ENA |
4956			  PACKET3_SH_KCACHE_ACTION_ENA |
4957			  PACKET3_SH_ICACHE_ACTION_ENA);  /* CP_COHER_CNTL */
4958	amdgpu_ring_write(ring, 0xffffffff);	/* CP_COHER_SIZE */
4959	amdgpu_ring_write(ring, 0xff);		/* CP_COHER_SIZE_HI */
4960	amdgpu_ring_write(ring, 0);		/* CP_COHER_BASE */
4961	amdgpu_ring_write(ring, 0);		/* CP_COHER_BASE_HI */
4962	amdgpu_ring_write(ring, 0x0000000A);	/* poll interval */
4963}
4964
4965static const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
4966	.name = "gfx_v7_0",
4967	.early_init = gfx_v7_0_early_init,
4968	.late_init = gfx_v7_0_late_init,
4969	.sw_init = gfx_v7_0_sw_init,
4970	.sw_fini = gfx_v7_0_sw_fini,
4971	.hw_init = gfx_v7_0_hw_init,
4972	.hw_fini = gfx_v7_0_hw_fini,
4973	.suspend = gfx_v7_0_suspend,
4974	.resume = gfx_v7_0_resume,
4975	.is_idle = gfx_v7_0_is_idle,
4976	.wait_for_idle = gfx_v7_0_wait_for_idle,
4977	.soft_reset = gfx_v7_0_soft_reset,
4978	.set_clockgating_state = gfx_v7_0_set_clockgating_state,
4979	.set_powergating_state = gfx_v7_0_set_powergating_state,
4980};
4981
4982static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
4983	.type = AMDGPU_RING_TYPE_GFX,
4984	.align_mask = 0xff,
4985	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
4986	.support_64bit_ptrs = false,
4987	.get_rptr = gfx_v7_0_ring_get_rptr,
4988	.get_wptr = gfx_v7_0_ring_get_wptr_gfx,
4989	.set_wptr = gfx_v7_0_ring_set_wptr_gfx,
4990	.emit_frame_size =
4991		20 + /* gfx_v7_0_ring_emit_gds_switch */
4992		7 + /* gfx_v7_0_ring_emit_hdp_flush */
4993		5 + /* hdp invalidate */
4994		12 + 12 + 12 + /* gfx_v7_0_ring_emit_fence_gfx x3 for user fence, vm fence */
4995		7 + 4 + /* gfx_v7_0_ring_emit_pipeline_sync */
4996		CIK_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v7_0_ring_emit_vm_flush */
4997		3 + 4 + /* gfx_v7_ring_emit_cntxcntl including vgt flush*/
4998		5, /* SURFACE_SYNC */
4999	.emit_ib_size = 4, /* gfx_v7_0_ring_emit_ib_gfx */
5000	.emit_ib = gfx_v7_0_ring_emit_ib_gfx,
5001	.emit_fence = gfx_v7_0_ring_emit_fence_gfx,
5002	.emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
5003	.emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
5004	.emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
5005	.emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
5006	.test_ring = gfx_v7_0_ring_test_ring,
5007	.test_ib = gfx_v7_0_ring_test_ib,
5008	.insert_nop = amdgpu_ring_insert_nop,
5009	.pad_ib = amdgpu_ring_generic_pad_ib,
5010	.emit_cntxcntl = gfx_v7_ring_emit_cntxcntl,
5011	.emit_wreg = gfx_v7_0_ring_emit_wreg,
5012	.soft_recovery = gfx_v7_0_ring_soft_recovery,
5013	.emit_mem_sync = gfx_v7_0_emit_mem_sync,
5014};
5015
5016static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
5017	.type = AMDGPU_RING_TYPE_COMPUTE,
5018	.align_mask = 0xff,
5019	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
5020	.support_64bit_ptrs = false,
5021	.get_rptr = gfx_v7_0_ring_get_rptr,
5022	.get_wptr = gfx_v7_0_ring_get_wptr_compute,
5023	.set_wptr = gfx_v7_0_ring_set_wptr_compute,
5024	.emit_frame_size =
5025		20 + /* gfx_v7_0_ring_emit_gds_switch */
5026		7 + /* gfx_v7_0_ring_emit_hdp_flush */
5027		5 + /* hdp invalidate */
5028		7 + /* gfx_v7_0_ring_emit_pipeline_sync */
5029		CIK_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v7_0_ring_emit_vm_flush */
5030		7 + 7 + 7 + /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */
5031		7, /* gfx_v7_0_emit_mem_sync_compute */
5032	.emit_ib_size =	7, /* gfx_v7_0_ring_emit_ib_compute */
5033	.emit_ib = gfx_v7_0_ring_emit_ib_compute,
5034	.emit_fence = gfx_v7_0_ring_emit_fence_compute,
5035	.emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
5036	.emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
5037	.emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
5038	.emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
5039	.test_ring = gfx_v7_0_ring_test_ring,
5040	.test_ib = gfx_v7_0_ring_test_ib,
5041	.insert_nop = amdgpu_ring_insert_nop,
5042	.pad_ib = amdgpu_ring_generic_pad_ib,
5043	.emit_wreg = gfx_v7_0_ring_emit_wreg,
5044	.emit_mem_sync = gfx_v7_0_emit_mem_sync_compute,
5045};
5046
5047static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev)
5048{
5049	int i;
5050
5051	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
5052		adev->gfx.gfx_ring[i].funcs = &gfx_v7_0_ring_funcs_gfx;
5053	for (i = 0; i < adev->gfx.num_compute_rings; i++)
5054		adev->gfx.compute_ring[i].funcs = &gfx_v7_0_ring_funcs_compute;
5055}
5056
5057static const struct amdgpu_irq_src_funcs gfx_v7_0_eop_irq_funcs = {
5058	.set = gfx_v7_0_set_eop_interrupt_state,
5059	.process = gfx_v7_0_eop_irq,
5060};
5061
5062static const struct amdgpu_irq_src_funcs gfx_v7_0_priv_reg_irq_funcs = {
5063	.set = gfx_v7_0_set_priv_reg_fault_state,
5064	.process = gfx_v7_0_priv_reg_irq,
5065};
5066
5067static const struct amdgpu_irq_src_funcs gfx_v7_0_priv_inst_irq_funcs = {
5068	.set = gfx_v7_0_set_priv_inst_fault_state,
5069	.process = gfx_v7_0_priv_inst_irq,
5070};
5071
5072static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev)
5073{
5074	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
5075	adev->gfx.eop_irq.funcs = &gfx_v7_0_eop_irq_funcs;
5076
5077	adev->gfx.priv_reg_irq.num_types = 1;
5078	adev->gfx.priv_reg_irq.funcs = &gfx_v7_0_priv_reg_irq_funcs;
5079
5080	adev->gfx.priv_inst_irq.num_types = 1;
5081	adev->gfx.priv_inst_irq.funcs = &gfx_v7_0_priv_inst_irq_funcs;
5082}
5083
5084static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev)
5085{
5086	/* init asci gds info */
5087	adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
5088	adev->gds.gws_size = 64;
5089	adev->gds.oa_size = 16;
5090	adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
5091}
5092
5093
5094static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
5095{
5096	int i, j, k, counter, active_cu_number = 0;
5097	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
5098	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
5099	unsigned disable_masks[4 * 2];
5100	u32 ao_cu_num;
5101
5102	if (adev->flags & AMD_IS_APU)
5103		ao_cu_num = 2;
5104	else
5105		ao_cu_num = adev->gfx.config.max_cu_per_sh;
5106
5107	memset(cu_info, 0, sizeof(*cu_info));
5108
5109	amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
5110
5111	mutex_lock(&adev->grbm_idx_mutex);
5112	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
5113		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
5114			mask = 1;
5115			ao_bitmap = 0;
5116			counter = 0;
5117			gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff, 0);
5118			if (i < 4 && j < 2)
5119				gfx_v7_0_set_user_cu_inactive_bitmap(
5120					adev, disable_masks[i * 2 + j]);
5121			bitmap = gfx_v7_0_get_cu_active_bitmap(adev);
5122			cu_info->bitmap[0][i][j] = bitmap;
5123
5124			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
5125				if (bitmap & mask) {
5126					if (counter < ao_cu_num)
5127						ao_bitmap |= mask;
5128					counter++;
5129				}
5130				mask <<= 1;
5131			}
5132			active_cu_number += counter;
5133			if (i < 2 && j < 2)
5134				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
5135			cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
5136		}
5137	}
5138	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
5139	mutex_unlock(&adev->grbm_idx_mutex);
5140
5141	cu_info->number = active_cu_number;
5142	cu_info->ao_cu_mask = ao_cu_mask;
5143	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
5144	cu_info->max_waves_per_simd = 10;
5145	cu_info->max_scratch_slots_per_cu = 32;
5146	cu_info->wave_front_size = 64;
5147	cu_info->lds_size = 64;
5148}
5149
5150const struct amdgpu_ip_block_version gfx_v7_1_ip_block = {
 
5151	.type = AMD_IP_BLOCK_TYPE_GFX,
5152	.major = 7,
5153	.minor = 1,
5154	.rev = 0,
5155	.funcs = &gfx_v7_0_ip_funcs,
5156};
5157
5158const struct amdgpu_ip_block_version gfx_v7_2_ip_block = {
 
5159	.type = AMD_IP_BLOCK_TYPE_GFX,
5160	.major = 7,
5161	.minor = 2,
5162	.rev = 0,
5163	.funcs = &gfx_v7_0_ip_funcs,
5164};
5165
5166const struct amdgpu_ip_block_version gfx_v7_3_ip_block = {
 
5167	.type = AMD_IP_BLOCK_TYPE_GFX,
5168	.major = 7,
5169	.minor = 3,
5170	.rev = 0,
5171	.funcs = &gfx_v7_0_ip_funcs,
5172};