1/*
   2 * Copyright 2011 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28#include <linux/slab.h>
  29
  30#include <drm/drm_vblank.h>
  31#include <drm/radeon_drm.h>
  32
  33#include "atom.h"
  34#include "clearstate_si.h"
  35#include "evergreen.h"
  36#include "r600.h"
  37#include "radeon.h"
  38#include "radeon_asic.h"
  39#include "radeon_audio.h"
  40#include "radeon_ucode.h"
  41#include "si_blit_shaders.h"
  42#include "si.h"
  43#include "sid.h"
  44
  45
  46MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  47MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  48MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  49MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  50MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  51MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  52MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  55MODULE_FIRMWARE("radeon/tahiti_me.bin");
  56MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  57MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  58MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  59MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  60
  61MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  62MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  63MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  64MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  65MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  66MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  67MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  68
  69MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  70MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  71MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  72MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  73MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  74MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
  75MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
  76
  77MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  78MODULE_FIRMWARE("radeon/VERDE_me.bin");
  79MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  80MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  81MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  82MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  83MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  84
  85MODULE_FIRMWARE("radeon/verde_pfp.bin");
  86MODULE_FIRMWARE("radeon/verde_me.bin");
  87MODULE_FIRMWARE("radeon/verde_ce.bin");
  88MODULE_FIRMWARE("radeon/verde_mc.bin");
  89MODULE_FIRMWARE("radeon/verde_rlc.bin");
  90MODULE_FIRMWARE("radeon/verde_smc.bin");
  91MODULE_FIRMWARE("radeon/verde_k_smc.bin");
  92
  93MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  94MODULE_FIRMWARE("radeon/OLAND_me.bin");
  95MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  96MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  97MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  98MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  99MODULE_FIRMWARE("radeon/OLAND_smc.bin");
 100
 101MODULE_FIRMWARE("radeon/oland_pfp.bin");
 102MODULE_FIRMWARE("radeon/oland_me.bin");
 103MODULE_FIRMWARE("radeon/oland_ce.bin");
 104MODULE_FIRMWARE("radeon/oland_mc.bin");
 105MODULE_FIRMWARE("radeon/oland_rlc.bin");
 106MODULE_FIRMWARE("radeon/oland_smc.bin");
 107MODULE_FIRMWARE("radeon/oland_k_smc.bin");
 108
 109MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 110MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 111MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 112MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 113MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 114MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 115MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 116
 117MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 118MODULE_FIRMWARE("radeon/hainan_me.bin");
 119MODULE_FIRMWARE("radeon/hainan_ce.bin");
 120MODULE_FIRMWARE("radeon/hainan_mc.bin");
 121MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 122MODULE_FIRMWARE("radeon/hainan_smc.bin");
 123MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
 124MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
 125
 126MODULE_FIRMWARE("radeon/si58_mc.bin");
 127
 128static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 129static void si_pcie_gen3_enable(struct radeon_device *rdev);
 130static void si_program_aspm(struct radeon_device *rdev);
 131extern void sumo_rlc_fini(struct radeon_device *rdev);
 132extern int sumo_rlc_init(struct radeon_device *rdev);
 133static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 134					 bool enable);
 135static void si_init_pg(struct radeon_device *rdev);
 136static void si_init_cg(struct radeon_device *rdev);
 137static void si_fini_pg(struct radeon_device *rdev);
 138static void si_fini_cg(struct radeon_device *rdev);
 139static void si_rlc_stop(struct radeon_device *rdev);
 140
 141static const u32 crtc_offsets[] = {
 
 142	EVERGREEN_CRTC0_REGISTER_OFFSET,
 143	EVERGREEN_CRTC1_REGISTER_OFFSET,
 144	EVERGREEN_CRTC2_REGISTER_OFFSET,
 145	EVERGREEN_CRTC3_REGISTER_OFFSET,
 146	EVERGREEN_CRTC4_REGISTER_OFFSET,
 147	EVERGREEN_CRTC5_REGISTER_OFFSET
 148};
 149
 150static const u32 si_disp_int_status[] = {
 
 151	DISP_INTERRUPT_STATUS,
 152	DISP_INTERRUPT_STATUS_CONTINUE,
 153	DISP_INTERRUPT_STATUS_CONTINUE2,
 154	DISP_INTERRUPT_STATUS_CONTINUE3,
 155	DISP_INTERRUPT_STATUS_CONTINUE4,
 156	DISP_INTERRUPT_STATUS_CONTINUE5
 157};
 158
 159#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
 160#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
 161#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
 162
 163static const u32 verde_rlc_save_restore_register_list[] = {
 
 164	(0x8000 << 16) | (0x98f4 >> 2),
 165	0x00000000,
 166	(0x8040 << 16) | (0x98f4 >> 2),
 167	0x00000000,
 168	(0x8000 << 16) | (0xe80 >> 2),
 169	0x00000000,
 170	(0x8040 << 16) | (0xe80 >> 2),
 171	0x00000000,
 172	(0x8000 << 16) | (0x89bc >> 2),
 173	0x00000000,
 174	(0x8040 << 16) | (0x89bc >> 2),
 175	0x00000000,
 176	(0x8000 << 16) | (0x8c1c >> 2),
 177	0x00000000,
 178	(0x8040 << 16) | (0x8c1c >> 2),
 179	0x00000000,
 180	(0x9c00 << 16) | (0x98f0 >> 2),
 181	0x00000000,
 182	(0x9c00 << 16) | (0xe7c >> 2),
 183	0x00000000,
 184	(0x8000 << 16) | (0x9148 >> 2),
 185	0x00000000,
 186	(0x8040 << 16) | (0x9148 >> 2),
 187	0x00000000,
 188	(0x9c00 << 16) | (0x9150 >> 2),
 189	0x00000000,
 190	(0x9c00 << 16) | (0x897c >> 2),
 191	0x00000000,
 192	(0x9c00 << 16) | (0x8d8c >> 2),
 193	0x00000000,
 194	(0x9c00 << 16) | (0xac54 >> 2),
 195	0X00000000,
 196	0x3,
 197	(0x9c00 << 16) | (0x98f8 >> 2),
 198	0x00000000,
 199	(0x9c00 << 16) | (0x9910 >> 2),
 200	0x00000000,
 201	(0x9c00 << 16) | (0x9914 >> 2),
 202	0x00000000,
 203	(0x9c00 << 16) | (0x9918 >> 2),
 204	0x00000000,
 205	(0x9c00 << 16) | (0x991c >> 2),
 206	0x00000000,
 207	(0x9c00 << 16) | (0x9920 >> 2),
 208	0x00000000,
 209	(0x9c00 << 16) | (0x9924 >> 2),
 210	0x00000000,
 211	(0x9c00 << 16) | (0x9928 >> 2),
 212	0x00000000,
 213	(0x9c00 << 16) | (0x992c >> 2),
 214	0x00000000,
 215	(0x9c00 << 16) | (0x9930 >> 2),
 216	0x00000000,
 217	(0x9c00 << 16) | (0x9934 >> 2),
 218	0x00000000,
 219	(0x9c00 << 16) | (0x9938 >> 2),
 220	0x00000000,
 221	(0x9c00 << 16) | (0x993c >> 2),
 222	0x00000000,
 223	(0x9c00 << 16) | (0x9940 >> 2),
 224	0x00000000,
 225	(0x9c00 << 16) | (0x9944 >> 2),
 226	0x00000000,
 227	(0x9c00 << 16) | (0x9948 >> 2),
 228	0x00000000,
 229	(0x9c00 << 16) | (0x994c >> 2),
 230	0x00000000,
 231	(0x9c00 << 16) | (0x9950 >> 2),
 232	0x00000000,
 233	(0x9c00 << 16) | (0x9954 >> 2),
 234	0x00000000,
 235	(0x9c00 << 16) | (0x9958 >> 2),
 236	0x00000000,
 237	(0x9c00 << 16) | (0x995c >> 2),
 238	0x00000000,
 239	(0x9c00 << 16) | (0x9960 >> 2),
 240	0x00000000,
 241	(0x9c00 << 16) | (0x9964 >> 2),
 242	0x00000000,
 243	(0x9c00 << 16) | (0x9968 >> 2),
 244	0x00000000,
 245	(0x9c00 << 16) | (0x996c >> 2),
 246	0x00000000,
 247	(0x9c00 << 16) | (0x9970 >> 2),
 248	0x00000000,
 249	(0x9c00 << 16) | (0x9974 >> 2),
 250	0x00000000,
 251	(0x9c00 << 16) | (0x9978 >> 2),
 252	0x00000000,
 253	(0x9c00 << 16) | (0x997c >> 2),
 254	0x00000000,
 255	(0x9c00 << 16) | (0x9980 >> 2),
 256	0x00000000,
 257	(0x9c00 << 16) | (0x9984 >> 2),
 258	0x00000000,
 259	(0x9c00 << 16) | (0x9988 >> 2),
 260	0x00000000,
 261	(0x9c00 << 16) | (0x998c >> 2),
 262	0x00000000,
 263	(0x9c00 << 16) | (0x8c00 >> 2),
 264	0x00000000,
 265	(0x9c00 << 16) | (0x8c14 >> 2),
 266	0x00000000,
 267	(0x9c00 << 16) | (0x8c04 >> 2),
 268	0x00000000,
 269	(0x9c00 << 16) | (0x8c08 >> 2),
 270	0x00000000,
 271	(0x8000 << 16) | (0x9b7c >> 2),
 272	0x00000000,
 273	(0x8040 << 16) | (0x9b7c >> 2),
 274	0x00000000,
 275	(0x8000 << 16) | (0xe84 >> 2),
 276	0x00000000,
 277	(0x8040 << 16) | (0xe84 >> 2),
 278	0x00000000,
 279	(0x8000 << 16) | (0x89c0 >> 2),
 280	0x00000000,
 281	(0x8040 << 16) | (0x89c0 >> 2),
 282	0x00000000,
 283	(0x8000 << 16) | (0x914c >> 2),
 284	0x00000000,
 285	(0x8040 << 16) | (0x914c >> 2),
 286	0x00000000,
 287	(0x8000 << 16) | (0x8c20 >> 2),
 288	0x00000000,
 289	(0x8040 << 16) | (0x8c20 >> 2),
 290	0x00000000,
 291	(0x8000 << 16) | (0x9354 >> 2),
 292	0x00000000,
 293	(0x8040 << 16) | (0x9354 >> 2),
 294	0x00000000,
 295	(0x9c00 << 16) | (0x9060 >> 2),
 296	0x00000000,
 297	(0x9c00 << 16) | (0x9364 >> 2),
 298	0x00000000,
 299	(0x9c00 << 16) | (0x9100 >> 2),
 300	0x00000000,
 301	(0x9c00 << 16) | (0x913c >> 2),
 302	0x00000000,
 303	(0x8000 << 16) | (0x90e0 >> 2),
 304	0x00000000,
 305	(0x8000 << 16) | (0x90e4 >> 2),
 306	0x00000000,
 307	(0x8000 << 16) | (0x90e8 >> 2),
 308	0x00000000,
 309	(0x8040 << 16) | (0x90e0 >> 2),
 310	0x00000000,
 311	(0x8040 << 16) | (0x90e4 >> 2),
 312	0x00000000,
 313	(0x8040 << 16) | (0x90e8 >> 2),
 314	0x00000000,
 315	(0x9c00 << 16) | (0x8bcc >> 2),
 316	0x00000000,
 317	(0x9c00 << 16) | (0x8b24 >> 2),
 318	0x00000000,
 319	(0x9c00 << 16) | (0x88c4 >> 2),
 320	0x00000000,
 321	(0x9c00 << 16) | (0x8e50 >> 2),
 322	0x00000000,
 323	(0x9c00 << 16) | (0x8c0c >> 2),
 324	0x00000000,
 325	(0x9c00 << 16) | (0x8e58 >> 2),
 326	0x00000000,
 327	(0x9c00 << 16) | (0x8e5c >> 2),
 328	0x00000000,
 329	(0x9c00 << 16) | (0x9508 >> 2),
 330	0x00000000,
 331	(0x9c00 << 16) | (0x950c >> 2),
 332	0x00000000,
 333	(0x9c00 << 16) | (0x9494 >> 2),
 334	0x00000000,
 335	(0x9c00 << 16) | (0xac0c >> 2),
 336	0x00000000,
 337	(0x9c00 << 16) | (0xac10 >> 2),
 338	0x00000000,
 339	(0x9c00 << 16) | (0xac14 >> 2),
 340	0x00000000,
 341	(0x9c00 << 16) | (0xae00 >> 2),
 342	0x00000000,
 343	(0x9c00 << 16) | (0xac08 >> 2),
 344	0x00000000,
 345	(0x9c00 << 16) | (0x88d4 >> 2),
 346	0x00000000,
 347	(0x9c00 << 16) | (0x88c8 >> 2),
 348	0x00000000,
 349	(0x9c00 << 16) | (0x88cc >> 2),
 350	0x00000000,
 351	(0x9c00 << 16) | (0x89b0 >> 2),
 352	0x00000000,
 353	(0x9c00 << 16) | (0x8b10 >> 2),
 354	0x00000000,
 355	(0x9c00 << 16) | (0x8a14 >> 2),
 356	0x00000000,
 357	(0x9c00 << 16) | (0x9830 >> 2),
 358	0x00000000,
 359	(0x9c00 << 16) | (0x9834 >> 2),
 360	0x00000000,
 361	(0x9c00 << 16) | (0x9838 >> 2),
 362	0x00000000,
 363	(0x9c00 << 16) | (0x9a10 >> 2),
 364	0x00000000,
 365	(0x8000 << 16) | (0x9870 >> 2),
 366	0x00000000,
 367	(0x8000 << 16) | (0x9874 >> 2),
 368	0x00000000,
 369	(0x8001 << 16) | (0x9870 >> 2),
 370	0x00000000,
 371	(0x8001 << 16) | (0x9874 >> 2),
 372	0x00000000,
 373	(0x8040 << 16) | (0x9870 >> 2),
 374	0x00000000,
 375	(0x8040 << 16) | (0x9874 >> 2),
 376	0x00000000,
 377	(0x8041 << 16) | (0x9870 >> 2),
 378	0x00000000,
 379	(0x8041 << 16) | (0x9874 >> 2),
 380	0x00000000,
 381	0x00000000
 382};
 383
 384static const u32 tahiti_golden_rlc_registers[] = {
 
 385	0xc424, 0xffffffff, 0x00601005,
 386	0xc47c, 0xffffffff, 0x10104040,
 387	0xc488, 0xffffffff, 0x0100000a,
 388	0xc314, 0xffffffff, 0x00000800,
 389	0xc30c, 0xffffffff, 0x800000f4,
 390	0xf4a8, 0xffffffff, 0x00000000
 391};
 392
 393static const u32 tahiti_golden_registers[] = {
 
 394	0x9a10, 0x00010000, 0x00018208,
 395	0x9830, 0xffffffff, 0x00000000,
 396	0x9834, 0xf00fffff, 0x00000400,
 397	0x9838, 0x0002021c, 0x00020200,
 398	0xc78, 0x00000080, 0x00000000,
 399	0xd030, 0x000300c0, 0x00800040,
 400	0xd830, 0x000300c0, 0x00800040,
 401	0x5bb0, 0x000000f0, 0x00000070,
 402	0x5bc0, 0x00200000, 0x50100000,
 403	0x7030, 0x31000311, 0x00000011,
 404	0x277c, 0x00000003, 0x000007ff,
 405	0x240c, 0x000007ff, 0x00000000,
 406	0x8a14, 0xf000001f, 0x00000007,
 407	0x8b24, 0xffffffff, 0x00ffffff,
 408	0x8b10, 0x0000ff0f, 0x00000000,
 409	0x28a4c, 0x07ffffff, 0x4e000000,
 410	0x28350, 0x3f3f3fff, 0x2a00126a,
 411	0x30, 0x000000ff, 0x0040,
 412	0x34, 0x00000040, 0x00004040,
 413	0x9100, 0x07ffffff, 0x03000000,
 414	0x8e88, 0x01ff1f3f, 0x00000000,
 415	0x8e84, 0x01ff1f3f, 0x00000000,
 416	0x9060, 0x0000007f, 0x00000020,
 417	0x9508, 0x00010000, 0x00010000,
 418	0xac14, 0x00000200, 0x000002fb,
 419	0xac10, 0xffffffff, 0x0000543b,
 420	0xac0c, 0xffffffff, 0xa9210876,
 421	0x88d0, 0xffffffff, 0x000fff40,
 422	0x88d4, 0x0000001f, 0x00000010,
 423	0x1410, 0x20000000, 0x20fffed8,
 424	0x15c0, 0x000c0fc0, 0x000c0400
 425};
 426
 427static const u32 tahiti_golden_registers2[] = {
 
 428	0xc64, 0x00000001, 0x00000001
 429};
 430
 431static const u32 pitcairn_golden_rlc_registers[] = {
 
 432	0xc424, 0xffffffff, 0x00601004,
 433	0xc47c, 0xffffffff, 0x10102020,
 434	0xc488, 0xffffffff, 0x01000020,
 435	0xc314, 0xffffffff, 0x00000800,
 436	0xc30c, 0xffffffff, 0x800000a4
 437};
 438
 439static const u32 pitcairn_golden_registers[] = {
 
 440	0x9a10, 0x00010000, 0x00018208,
 441	0x9830, 0xffffffff, 0x00000000,
 442	0x9834, 0xf00fffff, 0x00000400,
 443	0x9838, 0x0002021c, 0x00020200,
 444	0xc78, 0x00000080, 0x00000000,
 445	0xd030, 0x000300c0, 0x00800040,
 446	0xd830, 0x000300c0, 0x00800040,
 447	0x5bb0, 0x000000f0, 0x00000070,
 448	0x5bc0, 0x00200000, 0x50100000,
 449	0x7030, 0x31000311, 0x00000011,
 450	0x2ae4, 0x00073ffe, 0x000022a2,
 451	0x240c, 0x000007ff, 0x00000000,
 452	0x8a14, 0xf000001f, 0x00000007,
 453	0x8b24, 0xffffffff, 0x00ffffff,
 454	0x8b10, 0x0000ff0f, 0x00000000,
 455	0x28a4c, 0x07ffffff, 0x4e000000,
 456	0x28350, 0x3f3f3fff, 0x2a00126a,
 457	0x30, 0x000000ff, 0x0040,
 458	0x34, 0x00000040, 0x00004040,
 459	0x9100, 0x07ffffff, 0x03000000,
 460	0x9060, 0x0000007f, 0x00000020,
 461	0x9508, 0x00010000, 0x00010000,
 462	0xac14, 0x000003ff, 0x000000f7,
 463	0xac10, 0xffffffff, 0x00000000,
 464	0xac0c, 0xffffffff, 0x32761054,
 465	0x88d4, 0x0000001f, 0x00000010,
 466	0x15c0, 0x000c0fc0, 0x000c0400
 467};
 468
 469static const u32 verde_golden_rlc_registers[] = {
 
 470	0xc424, 0xffffffff, 0x033f1005,
 471	0xc47c, 0xffffffff, 0x10808020,
 472	0xc488, 0xffffffff, 0x00800008,
 473	0xc314, 0xffffffff, 0x00001000,
 474	0xc30c, 0xffffffff, 0x80010014
 475};
 476
 477static const u32 verde_golden_registers[] = {
 
 478	0x9a10, 0x00010000, 0x00018208,
 479	0x9830, 0xffffffff, 0x00000000,
 480	0x9834, 0xf00fffff, 0x00000400,
 481	0x9838, 0x0002021c, 0x00020200,
 482	0xc78, 0x00000080, 0x00000000,
 483	0xd030, 0x000300c0, 0x00800040,
 484	0xd030, 0x000300c0, 0x00800040,
 485	0xd830, 0x000300c0, 0x00800040,
 486	0xd830, 0x000300c0, 0x00800040,
 487	0x5bb0, 0x000000f0, 0x00000070,
 488	0x5bc0, 0x00200000, 0x50100000,
 489	0x7030, 0x31000311, 0x00000011,
 490	0x2ae4, 0x00073ffe, 0x000022a2,
 491	0x2ae4, 0x00073ffe, 0x000022a2,
 492	0x2ae4, 0x00073ffe, 0x000022a2,
 493	0x240c, 0x000007ff, 0x00000000,
 494	0x240c, 0x000007ff, 0x00000000,
 495	0x240c, 0x000007ff, 0x00000000,
 496	0x8a14, 0xf000001f, 0x00000007,
 497	0x8a14, 0xf000001f, 0x00000007,
 498	0x8a14, 0xf000001f, 0x00000007,
 499	0x8b24, 0xffffffff, 0x00ffffff,
 500	0x8b10, 0x0000ff0f, 0x00000000,
 501	0x28a4c, 0x07ffffff, 0x4e000000,
 502	0x28350, 0x3f3f3fff, 0x0000124a,
 503	0x28350, 0x3f3f3fff, 0x0000124a,
 504	0x28350, 0x3f3f3fff, 0x0000124a,
 505	0x30, 0x000000ff, 0x0040,
 506	0x34, 0x00000040, 0x00004040,
 507	0x9100, 0x07ffffff, 0x03000000,
 508	0x9100, 0x07ffffff, 0x03000000,
 509	0x8e88, 0x01ff1f3f, 0x00000000,
 510	0x8e88, 0x01ff1f3f, 0x00000000,
 511	0x8e88, 0x01ff1f3f, 0x00000000,
 512	0x8e84, 0x01ff1f3f, 0x00000000,
 513	0x8e84, 0x01ff1f3f, 0x00000000,
 514	0x8e84, 0x01ff1f3f, 0x00000000,
 515	0x9060, 0x0000007f, 0x00000020,
 516	0x9508, 0x00010000, 0x00010000,
 517	0xac14, 0x000003ff, 0x00000003,
 518	0xac14, 0x000003ff, 0x00000003,
 519	0xac14, 0x000003ff, 0x00000003,
 520	0xac10, 0xffffffff, 0x00000000,
 521	0xac10, 0xffffffff, 0x00000000,
 522	0xac10, 0xffffffff, 0x00000000,
 523	0xac0c, 0xffffffff, 0x00001032,
 524	0xac0c, 0xffffffff, 0x00001032,
 525	0xac0c, 0xffffffff, 0x00001032,
 526	0x88d4, 0x0000001f, 0x00000010,
 527	0x88d4, 0x0000001f, 0x00000010,
 528	0x88d4, 0x0000001f, 0x00000010,
 529	0x15c0, 0x000c0fc0, 0x000c0400
 530};
 531
 532static const u32 oland_golden_rlc_registers[] = {
 
 533	0xc424, 0xffffffff, 0x00601005,
 534	0xc47c, 0xffffffff, 0x10104040,
 535	0xc488, 0xffffffff, 0x0100000a,
 536	0xc314, 0xffffffff, 0x00000800,
 537	0xc30c, 0xffffffff, 0x800000f4
 538};
 539
 540static const u32 oland_golden_registers[] = {
 
 541	0x9a10, 0x00010000, 0x00018208,
 542	0x9830, 0xffffffff, 0x00000000,
 543	0x9834, 0xf00fffff, 0x00000400,
 544	0x9838, 0x0002021c, 0x00020200,
 545	0xc78, 0x00000080, 0x00000000,
 546	0xd030, 0x000300c0, 0x00800040,
 547	0xd830, 0x000300c0, 0x00800040,
 548	0x5bb0, 0x000000f0, 0x00000070,
 549	0x5bc0, 0x00200000, 0x50100000,
 550	0x7030, 0x31000311, 0x00000011,
 551	0x2ae4, 0x00073ffe, 0x000022a2,
 552	0x240c, 0x000007ff, 0x00000000,
 553	0x8a14, 0xf000001f, 0x00000007,
 554	0x8b24, 0xffffffff, 0x00ffffff,
 555	0x8b10, 0x0000ff0f, 0x00000000,
 556	0x28a4c, 0x07ffffff, 0x4e000000,
 557	0x28350, 0x3f3f3fff, 0x00000082,
 558	0x30, 0x000000ff, 0x0040,
 559	0x34, 0x00000040, 0x00004040,
 560	0x9100, 0x07ffffff, 0x03000000,
 561	0x9060, 0x0000007f, 0x00000020,
 562	0x9508, 0x00010000, 0x00010000,
 563	0xac14, 0x000003ff, 0x000000f3,
 564	0xac10, 0xffffffff, 0x00000000,
 565	0xac0c, 0xffffffff, 0x00003210,
 566	0x88d4, 0x0000001f, 0x00000010,
 567	0x15c0, 0x000c0fc0, 0x000c0400
 568};
 569
 570static const u32 hainan_golden_registers[] = {
 
 571	0x9a10, 0x00010000, 0x00018208,
 572	0x9830, 0xffffffff, 0x00000000,
 573	0x9834, 0xf00fffff, 0x00000400,
 574	0x9838, 0x0002021c, 0x00020200,
 575	0xd0c0, 0xff000fff, 0x00000100,
 576	0xd030, 0x000300c0, 0x00800040,
 577	0xd8c0, 0xff000fff, 0x00000100,
 578	0xd830, 0x000300c0, 0x00800040,
 579	0x2ae4, 0x00073ffe, 0x000022a2,
 580	0x240c, 0x000007ff, 0x00000000,
 581	0x8a14, 0xf000001f, 0x00000007,
 582	0x8b24, 0xffffffff, 0x00ffffff,
 583	0x8b10, 0x0000ff0f, 0x00000000,
 584	0x28a4c, 0x07ffffff, 0x4e000000,
 585	0x28350, 0x3f3f3fff, 0x00000000,
 586	0x30, 0x000000ff, 0x0040,
 587	0x34, 0x00000040, 0x00004040,
 588	0x9100, 0x03e00000, 0x03600000,
 589	0x9060, 0x0000007f, 0x00000020,
 590	0x9508, 0x00010000, 0x00010000,
 591	0xac14, 0x000003ff, 0x000000f1,
 592	0xac10, 0xffffffff, 0x00000000,
 593	0xac0c, 0xffffffff, 0x00003210,
 594	0x88d4, 0x0000001f, 0x00000010,
 595	0x15c0, 0x000c0fc0, 0x000c0400
 596};
 597
 598static const u32 hainan_golden_registers2[] = {
 
 599	0x98f8, 0xffffffff, 0x02010001
 600};
 601
 602static const u32 tahiti_mgcg_cgcg_init[] = {
 
 603	0xc400, 0xffffffff, 0xfffffffc,
 604	0x802c, 0xffffffff, 0xe0000000,
 605	0x9a60, 0xffffffff, 0x00000100,
 606	0x92a4, 0xffffffff, 0x00000100,
 607	0xc164, 0xffffffff, 0x00000100,
 608	0x9774, 0xffffffff, 0x00000100,
 609	0x8984, 0xffffffff, 0x06000100,
 610	0x8a18, 0xffffffff, 0x00000100,
 611	0x92a0, 0xffffffff, 0x00000100,
 612	0xc380, 0xffffffff, 0x00000100,
 613	0x8b28, 0xffffffff, 0x00000100,
 614	0x9144, 0xffffffff, 0x00000100,
 615	0x8d88, 0xffffffff, 0x00000100,
 616	0x8d8c, 0xffffffff, 0x00000100,
 617	0x9030, 0xffffffff, 0x00000100,
 618	0x9034, 0xffffffff, 0x00000100,
 619	0x9038, 0xffffffff, 0x00000100,
 620	0x903c, 0xffffffff, 0x00000100,
 621	0xad80, 0xffffffff, 0x00000100,
 622	0xac54, 0xffffffff, 0x00000100,
 623	0x897c, 0xffffffff, 0x06000100,
 624	0x9868, 0xffffffff, 0x00000100,
 625	0x9510, 0xffffffff, 0x00000100,
 626	0xaf04, 0xffffffff, 0x00000100,
 627	0xae04, 0xffffffff, 0x00000100,
 628	0x949c, 0xffffffff, 0x00000100,
 629	0x802c, 0xffffffff, 0xe0000000,
 630	0x9160, 0xffffffff, 0x00010000,
 631	0x9164, 0xffffffff, 0x00030002,
 632	0x9168, 0xffffffff, 0x00040007,
 633	0x916c, 0xffffffff, 0x00060005,
 634	0x9170, 0xffffffff, 0x00090008,
 635	0x9174, 0xffffffff, 0x00020001,
 636	0x9178, 0xffffffff, 0x00040003,
 637	0x917c, 0xffffffff, 0x00000007,
 638	0x9180, 0xffffffff, 0x00060005,
 639	0x9184, 0xffffffff, 0x00090008,
 640	0x9188, 0xffffffff, 0x00030002,
 641	0x918c, 0xffffffff, 0x00050004,
 642	0x9190, 0xffffffff, 0x00000008,
 643	0x9194, 0xffffffff, 0x00070006,
 644	0x9198, 0xffffffff, 0x000a0009,
 645	0x919c, 0xffffffff, 0x00040003,
 646	0x91a0, 0xffffffff, 0x00060005,
 647	0x91a4, 0xffffffff, 0x00000009,
 648	0x91a8, 0xffffffff, 0x00080007,
 649	0x91ac, 0xffffffff, 0x000b000a,
 650	0x91b0, 0xffffffff, 0x00050004,
 651	0x91b4, 0xffffffff, 0x00070006,
 652	0x91b8, 0xffffffff, 0x0008000b,
 653	0x91bc, 0xffffffff, 0x000a0009,
 654	0x91c0, 0xffffffff, 0x000d000c,
 655	0x91c4, 0xffffffff, 0x00060005,
 656	0x91c8, 0xffffffff, 0x00080007,
 657	0x91cc, 0xffffffff, 0x0000000b,
 658	0x91d0, 0xffffffff, 0x000a0009,
 659	0x91d4, 0xffffffff, 0x000d000c,
 660	0x91d8, 0xffffffff, 0x00070006,
 661	0x91dc, 0xffffffff, 0x00090008,
 662	0x91e0, 0xffffffff, 0x0000000c,
 663	0x91e4, 0xffffffff, 0x000b000a,
 664	0x91e8, 0xffffffff, 0x000e000d,
 665	0x91ec, 0xffffffff, 0x00080007,
 666	0x91f0, 0xffffffff, 0x000a0009,
 667	0x91f4, 0xffffffff, 0x0000000d,
 668	0x91f8, 0xffffffff, 0x000c000b,
 669	0x91fc, 0xffffffff, 0x000f000e,
 670	0x9200, 0xffffffff, 0x00090008,
 671	0x9204, 0xffffffff, 0x000b000a,
 672	0x9208, 0xffffffff, 0x000c000f,
 673	0x920c, 0xffffffff, 0x000e000d,
 674	0x9210, 0xffffffff, 0x00110010,
 675	0x9214, 0xffffffff, 0x000a0009,
 676	0x9218, 0xffffffff, 0x000c000b,
 677	0x921c, 0xffffffff, 0x0000000f,
 678	0x9220, 0xffffffff, 0x000e000d,
 679	0x9224, 0xffffffff, 0x00110010,
 680	0x9228, 0xffffffff, 0x000b000a,
 681	0x922c, 0xffffffff, 0x000d000c,
 682	0x9230, 0xffffffff, 0x00000010,
 683	0x9234, 0xffffffff, 0x000f000e,
 684	0x9238, 0xffffffff, 0x00120011,
 685	0x923c, 0xffffffff, 0x000c000b,
 686	0x9240, 0xffffffff, 0x000e000d,
 687	0x9244, 0xffffffff, 0x00000011,
 688	0x9248, 0xffffffff, 0x0010000f,
 689	0x924c, 0xffffffff, 0x00130012,
 690	0x9250, 0xffffffff, 0x000d000c,
 691	0x9254, 0xffffffff, 0x000f000e,
 692	0x9258, 0xffffffff, 0x00100013,
 693	0x925c, 0xffffffff, 0x00120011,
 694	0x9260, 0xffffffff, 0x00150014,
 695	0x9264, 0xffffffff, 0x000e000d,
 696	0x9268, 0xffffffff, 0x0010000f,
 697	0x926c, 0xffffffff, 0x00000013,
 698	0x9270, 0xffffffff, 0x00120011,
 699	0x9274, 0xffffffff, 0x00150014,
 700	0x9278, 0xffffffff, 0x000f000e,
 701	0x927c, 0xffffffff, 0x00110010,
 702	0x9280, 0xffffffff, 0x00000014,
 703	0x9284, 0xffffffff, 0x00130012,
 704	0x9288, 0xffffffff, 0x00160015,
 705	0x928c, 0xffffffff, 0x0010000f,
 706	0x9290, 0xffffffff, 0x00120011,
 707	0x9294, 0xffffffff, 0x00000015,
 708	0x9298, 0xffffffff, 0x00140013,
 709	0x929c, 0xffffffff, 0x00170016,
 710	0x9150, 0xffffffff, 0x96940200,
 711	0x8708, 0xffffffff, 0x00900100,
 712	0xc478, 0xffffffff, 0x00000080,
 713	0xc404, 0xffffffff, 0x0020003f,
 714	0x30, 0xffffffff, 0x0000001c,
 715	0x34, 0x000f0000, 0x000f0000,
 716	0x160c, 0xffffffff, 0x00000100,
 717	0x1024, 0xffffffff, 0x00000100,
 718	0x102c, 0x00000101, 0x00000000,
 719	0x20a8, 0xffffffff, 0x00000104,
 720	0x264c, 0x000c0000, 0x000c0000,
 721	0x2648, 0x000c0000, 0x000c0000,
 722	0x55e4, 0xff000fff, 0x00000100,
 723	0x55e8, 0x00000001, 0x00000001,
 724	0x2f50, 0x00000001, 0x00000001,
 725	0x30cc, 0xc0000fff, 0x00000104,
 726	0xc1e4, 0x00000001, 0x00000001,
 727	0xd0c0, 0xfffffff0, 0x00000100,
 728	0xd8c0, 0xfffffff0, 0x00000100
 729};
 730
 731static const u32 pitcairn_mgcg_cgcg_init[] = {
 
 732	0xc400, 0xffffffff, 0xfffffffc,
 733	0x802c, 0xffffffff, 0xe0000000,
 734	0x9a60, 0xffffffff, 0x00000100,
 735	0x92a4, 0xffffffff, 0x00000100,
 736	0xc164, 0xffffffff, 0x00000100,
 737	0x9774, 0xffffffff, 0x00000100,
 738	0x8984, 0xffffffff, 0x06000100,
 739	0x8a18, 0xffffffff, 0x00000100,
 740	0x92a0, 0xffffffff, 0x00000100,
 741	0xc380, 0xffffffff, 0x00000100,
 742	0x8b28, 0xffffffff, 0x00000100,
 743	0x9144, 0xffffffff, 0x00000100,
 744	0x8d88, 0xffffffff, 0x00000100,
 745	0x8d8c, 0xffffffff, 0x00000100,
 746	0x9030, 0xffffffff, 0x00000100,
 747	0x9034, 0xffffffff, 0x00000100,
 748	0x9038, 0xffffffff, 0x00000100,
 749	0x903c, 0xffffffff, 0x00000100,
 750	0xad80, 0xffffffff, 0x00000100,
 751	0xac54, 0xffffffff, 0x00000100,
 752	0x897c, 0xffffffff, 0x06000100,
 753	0x9868, 0xffffffff, 0x00000100,
 754	0x9510, 0xffffffff, 0x00000100,
 755	0xaf04, 0xffffffff, 0x00000100,
 756	0xae04, 0xffffffff, 0x00000100,
 757	0x949c, 0xffffffff, 0x00000100,
 758	0x802c, 0xffffffff, 0xe0000000,
 759	0x9160, 0xffffffff, 0x00010000,
 760	0x9164, 0xffffffff, 0x00030002,
 761	0x9168, 0xffffffff, 0x00040007,
 762	0x916c, 0xffffffff, 0x00060005,
 763	0x9170, 0xffffffff, 0x00090008,
 764	0x9174, 0xffffffff, 0x00020001,
 765	0x9178, 0xffffffff, 0x00040003,
 766	0x917c, 0xffffffff, 0x00000007,
 767	0x9180, 0xffffffff, 0x00060005,
 768	0x9184, 0xffffffff, 0x00090008,
 769	0x9188, 0xffffffff, 0x00030002,
 770	0x918c, 0xffffffff, 0x00050004,
 771	0x9190, 0xffffffff, 0x00000008,
 772	0x9194, 0xffffffff, 0x00070006,
 773	0x9198, 0xffffffff, 0x000a0009,
 774	0x919c, 0xffffffff, 0x00040003,
 775	0x91a0, 0xffffffff, 0x00060005,
 776	0x91a4, 0xffffffff, 0x00000009,
 777	0x91a8, 0xffffffff, 0x00080007,
 778	0x91ac, 0xffffffff, 0x000b000a,
 779	0x91b0, 0xffffffff, 0x00050004,
 780	0x91b4, 0xffffffff, 0x00070006,
 781	0x91b8, 0xffffffff, 0x0008000b,
 782	0x91bc, 0xffffffff, 0x000a0009,
 783	0x91c0, 0xffffffff, 0x000d000c,
 784	0x9200, 0xffffffff, 0x00090008,
 785	0x9204, 0xffffffff, 0x000b000a,
 786	0x9208, 0xffffffff, 0x000c000f,
 787	0x920c, 0xffffffff, 0x000e000d,
 788	0x9210, 0xffffffff, 0x00110010,
 789	0x9214, 0xffffffff, 0x000a0009,
 790	0x9218, 0xffffffff, 0x000c000b,
 791	0x921c, 0xffffffff, 0x0000000f,
 792	0x9220, 0xffffffff, 0x000e000d,
 793	0x9224, 0xffffffff, 0x00110010,
 794	0x9228, 0xffffffff, 0x000b000a,
 795	0x922c, 0xffffffff, 0x000d000c,
 796	0x9230, 0xffffffff, 0x00000010,
 797	0x9234, 0xffffffff, 0x000f000e,
 798	0x9238, 0xffffffff, 0x00120011,
 799	0x923c, 0xffffffff, 0x000c000b,
 800	0x9240, 0xffffffff, 0x000e000d,
 801	0x9244, 0xffffffff, 0x00000011,
 802	0x9248, 0xffffffff, 0x0010000f,
 803	0x924c, 0xffffffff, 0x00130012,
 804	0x9250, 0xffffffff, 0x000d000c,
 805	0x9254, 0xffffffff, 0x000f000e,
 806	0x9258, 0xffffffff, 0x00100013,
 807	0x925c, 0xffffffff, 0x00120011,
 808	0x9260, 0xffffffff, 0x00150014,
 809	0x9150, 0xffffffff, 0x96940200,
 810	0x8708, 0xffffffff, 0x00900100,
 811	0xc478, 0xffffffff, 0x00000080,
 812	0xc404, 0xffffffff, 0x0020003f,
 813	0x30, 0xffffffff, 0x0000001c,
 814	0x34, 0x000f0000, 0x000f0000,
 815	0x160c, 0xffffffff, 0x00000100,
 816	0x1024, 0xffffffff, 0x00000100,
 817	0x102c, 0x00000101, 0x00000000,
 818	0x20a8, 0xffffffff, 0x00000104,
 819	0x55e4, 0xff000fff, 0x00000100,
 820	0x55e8, 0x00000001, 0x00000001,
 821	0x2f50, 0x00000001, 0x00000001,
 822	0x30cc, 0xc0000fff, 0x00000104,
 823	0xc1e4, 0x00000001, 0x00000001,
 824	0xd0c0, 0xfffffff0, 0x00000100,
 825	0xd8c0, 0xfffffff0, 0x00000100
 826};
 827
 828static const u32 verde_mgcg_cgcg_init[] = {
 
 829	0xc400, 0xffffffff, 0xfffffffc,
 830	0x802c, 0xffffffff, 0xe0000000,
 831	0x9a60, 0xffffffff, 0x00000100,
 832	0x92a4, 0xffffffff, 0x00000100,
 833	0xc164, 0xffffffff, 0x00000100,
 834	0x9774, 0xffffffff, 0x00000100,
 835	0x8984, 0xffffffff, 0x06000100,
 836	0x8a18, 0xffffffff, 0x00000100,
 837	0x92a0, 0xffffffff, 0x00000100,
 838	0xc380, 0xffffffff, 0x00000100,
 839	0x8b28, 0xffffffff, 0x00000100,
 840	0x9144, 0xffffffff, 0x00000100,
 841	0x8d88, 0xffffffff, 0x00000100,
 842	0x8d8c, 0xffffffff, 0x00000100,
 843	0x9030, 0xffffffff, 0x00000100,
 844	0x9034, 0xffffffff, 0x00000100,
 845	0x9038, 0xffffffff, 0x00000100,
 846	0x903c, 0xffffffff, 0x00000100,
 847	0xad80, 0xffffffff, 0x00000100,
 848	0xac54, 0xffffffff, 0x00000100,
 849	0x897c, 0xffffffff, 0x06000100,
 850	0x9868, 0xffffffff, 0x00000100,
 851	0x9510, 0xffffffff, 0x00000100,
 852	0xaf04, 0xffffffff, 0x00000100,
 853	0xae04, 0xffffffff, 0x00000100,
 854	0x949c, 0xffffffff, 0x00000100,
 855	0x802c, 0xffffffff, 0xe0000000,
 856	0x9160, 0xffffffff, 0x00010000,
 857	0x9164, 0xffffffff, 0x00030002,
 858	0x9168, 0xffffffff, 0x00040007,
 859	0x916c, 0xffffffff, 0x00060005,
 860	0x9170, 0xffffffff, 0x00090008,
 861	0x9174, 0xffffffff, 0x00020001,
 862	0x9178, 0xffffffff, 0x00040003,
 863	0x917c, 0xffffffff, 0x00000007,
 864	0x9180, 0xffffffff, 0x00060005,
 865	0x9184, 0xffffffff, 0x00090008,
 866	0x9188, 0xffffffff, 0x00030002,
 867	0x918c, 0xffffffff, 0x00050004,
 868	0x9190, 0xffffffff, 0x00000008,
 869	0x9194, 0xffffffff, 0x00070006,
 870	0x9198, 0xffffffff, 0x000a0009,
 871	0x919c, 0xffffffff, 0x00040003,
 872	0x91a0, 0xffffffff, 0x00060005,
 873	0x91a4, 0xffffffff, 0x00000009,
 874	0x91a8, 0xffffffff, 0x00080007,
 875	0x91ac, 0xffffffff, 0x000b000a,
 876	0x91b0, 0xffffffff, 0x00050004,
 877	0x91b4, 0xffffffff, 0x00070006,
 878	0x91b8, 0xffffffff, 0x0008000b,
 879	0x91bc, 0xffffffff, 0x000a0009,
 880	0x91c0, 0xffffffff, 0x000d000c,
 881	0x9200, 0xffffffff, 0x00090008,
 882	0x9204, 0xffffffff, 0x000b000a,
 883	0x9208, 0xffffffff, 0x000c000f,
 884	0x920c, 0xffffffff, 0x000e000d,
 885	0x9210, 0xffffffff, 0x00110010,
 886	0x9214, 0xffffffff, 0x000a0009,
 887	0x9218, 0xffffffff, 0x000c000b,
 888	0x921c, 0xffffffff, 0x0000000f,
 889	0x9220, 0xffffffff, 0x000e000d,
 890	0x9224, 0xffffffff, 0x00110010,
 891	0x9228, 0xffffffff, 0x000b000a,
 892	0x922c, 0xffffffff, 0x000d000c,
 893	0x9230, 0xffffffff, 0x00000010,
 894	0x9234, 0xffffffff, 0x000f000e,
 895	0x9238, 0xffffffff, 0x00120011,
 896	0x923c, 0xffffffff, 0x000c000b,
 897	0x9240, 0xffffffff, 0x000e000d,
 898	0x9244, 0xffffffff, 0x00000011,
 899	0x9248, 0xffffffff, 0x0010000f,
 900	0x924c, 0xffffffff, 0x00130012,
 901	0x9250, 0xffffffff, 0x000d000c,
 902	0x9254, 0xffffffff, 0x000f000e,
 903	0x9258, 0xffffffff, 0x00100013,
 904	0x925c, 0xffffffff, 0x00120011,
 905	0x9260, 0xffffffff, 0x00150014,
 906	0x9150, 0xffffffff, 0x96940200,
 907	0x8708, 0xffffffff, 0x00900100,
 908	0xc478, 0xffffffff, 0x00000080,
 909	0xc404, 0xffffffff, 0x0020003f,
 910	0x30, 0xffffffff, 0x0000001c,
 911	0x34, 0x000f0000, 0x000f0000,
 912	0x160c, 0xffffffff, 0x00000100,
 913	0x1024, 0xffffffff, 0x00000100,
 914	0x102c, 0x00000101, 0x00000000,
 915	0x20a8, 0xffffffff, 0x00000104,
 916	0x264c, 0x000c0000, 0x000c0000,
 917	0x2648, 0x000c0000, 0x000c0000,
 918	0x55e4, 0xff000fff, 0x00000100,
 919	0x55e8, 0x00000001, 0x00000001,
 920	0x2f50, 0x00000001, 0x00000001,
 921	0x30cc, 0xc0000fff, 0x00000104,
 922	0xc1e4, 0x00000001, 0x00000001,
 923	0xd0c0, 0xfffffff0, 0x00000100,
 924	0xd8c0, 0xfffffff0, 0x00000100
 925};
 926
 927static const u32 oland_mgcg_cgcg_init[] = {
 
 928	0xc400, 0xffffffff, 0xfffffffc,
 929	0x802c, 0xffffffff, 0xe0000000,
 930	0x9a60, 0xffffffff, 0x00000100,
 931	0x92a4, 0xffffffff, 0x00000100,
 932	0xc164, 0xffffffff, 0x00000100,
 933	0x9774, 0xffffffff, 0x00000100,
 934	0x8984, 0xffffffff, 0x06000100,
 935	0x8a18, 0xffffffff, 0x00000100,
 936	0x92a0, 0xffffffff, 0x00000100,
 937	0xc380, 0xffffffff, 0x00000100,
 938	0x8b28, 0xffffffff, 0x00000100,
 939	0x9144, 0xffffffff, 0x00000100,
 940	0x8d88, 0xffffffff, 0x00000100,
 941	0x8d8c, 0xffffffff, 0x00000100,
 942	0x9030, 0xffffffff, 0x00000100,
 943	0x9034, 0xffffffff, 0x00000100,
 944	0x9038, 0xffffffff, 0x00000100,
 945	0x903c, 0xffffffff, 0x00000100,
 946	0xad80, 0xffffffff, 0x00000100,
 947	0xac54, 0xffffffff, 0x00000100,
 948	0x897c, 0xffffffff, 0x06000100,
 949	0x9868, 0xffffffff, 0x00000100,
 950	0x9510, 0xffffffff, 0x00000100,
 951	0xaf04, 0xffffffff, 0x00000100,
 952	0xae04, 0xffffffff, 0x00000100,
 953	0x949c, 0xffffffff, 0x00000100,
 954	0x802c, 0xffffffff, 0xe0000000,
 955	0x9160, 0xffffffff, 0x00010000,
 956	0x9164, 0xffffffff, 0x00030002,
 957	0x9168, 0xffffffff, 0x00040007,
 958	0x916c, 0xffffffff, 0x00060005,
 959	0x9170, 0xffffffff, 0x00090008,
 960	0x9174, 0xffffffff, 0x00020001,
 961	0x9178, 0xffffffff, 0x00040003,
 962	0x917c, 0xffffffff, 0x00000007,
 963	0x9180, 0xffffffff, 0x00060005,
 964	0x9184, 0xffffffff, 0x00090008,
 965	0x9188, 0xffffffff, 0x00030002,
 966	0x918c, 0xffffffff, 0x00050004,
 967	0x9190, 0xffffffff, 0x00000008,
 968	0x9194, 0xffffffff, 0x00070006,
 969	0x9198, 0xffffffff, 0x000a0009,
 970	0x919c, 0xffffffff, 0x00040003,
 971	0x91a0, 0xffffffff, 0x00060005,
 972	0x91a4, 0xffffffff, 0x00000009,
 973	0x91a8, 0xffffffff, 0x00080007,
 974	0x91ac, 0xffffffff, 0x000b000a,
 975	0x91b0, 0xffffffff, 0x00050004,
 976	0x91b4, 0xffffffff, 0x00070006,
 977	0x91b8, 0xffffffff, 0x0008000b,
 978	0x91bc, 0xffffffff, 0x000a0009,
 979	0x91c0, 0xffffffff, 0x000d000c,
 980	0x91c4, 0xffffffff, 0x00060005,
 981	0x91c8, 0xffffffff, 0x00080007,
 982	0x91cc, 0xffffffff, 0x0000000b,
 983	0x91d0, 0xffffffff, 0x000a0009,
 984	0x91d4, 0xffffffff, 0x000d000c,
 985	0x9150, 0xffffffff, 0x96940200,
 986	0x8708, 0xffffffff, 0x00900100,
 987	0xc478, 0xffffffff, 0x00000080,
 988	0xc404, 0xffffffff, 0x0020003f,
 989	0x30, 0xffffffff, 0x0000001c,
 990	0x34, 0x000f0000, 0x000f0000,
 991	0x160c, 0xffffffff, 0x00000100,
 992	0x1024, 0xffffffff, 0x00000100,
 993	0x102c, 0x00000101, 0x00000000,
 994	0x20a8, 0xffffffff, 0x00000104,
 995	0x264c, 0x000c0000, 0x000c0000,
 996	0x2648, 0x000c0000, 0x000c0000,
 997	0x55e4, 0xff000fff, 0x00000100,
 998	0x55e8, 0x00000001, 0x00000001,
 999	0x2f50, 0x00000001, 0x00000001,
1000	0x30cc, 0xc0000fff, 0x00000104,
1001	0xc1e4, 0x00000001, 0x00000001,
1002	0xd0c0, 0xfffffff0, 0x00000100,
1003	0xd8c0, 0xfffffff0, 0x00000100
1004};
1005
1006static const u32 hainan_mgcg_cgcg_init[] = {
 
1007	0xc400, 0xffffffff, 0xfffffffc,
1008	0x802c, 0xffffffff, 0xe0000000,
1009	0x9a60, 0xffffffff, 0x00000100,
1010	0x92a4, 0xffffffff, 0x00000100,
1011	0xc164, 0xffffffff, 0x00000100,
1012	0x9774, 0xffffffff, 0x00000100,
1013	0x8984, 0xffffffff, 0x06000100,
1014	0x8a18, 0xffffffff, 0x00000100,
1015	0x92a0, 0xffffffff, 0x00000100,
1016	0xc380, 0xffffffff, 0x00000100,
1017	0x8b28, 0xffffffff, 0x00000100,
1018	0x9144, 0xffffffff, 0x00000100,
1019	0x8d88, 0xffffffff, 0x00000100,
1020	0x8d8c, 0xffffffff, 0x00000100,
1021	0x9030, 0xffffffff, 0x00000100,
1022	0x9034, 0xffffffff, 0x00000100,
1023	0x9038, 0xffffffff, 0x00000100,
1024	0x903c, 0xffffffff, 0x00000100,
1025	0xad80, 0xffffffff, 0x00000100,
1026	0xac54, 0xffffffff, 0x00000100,
1027	0x897c, 0xffffffff, 0x06000100,
1028	0x9868, 0xffffffff, 0x00000100,
1029	0x9510, 0xffffffff, 0x00000100,
1030	0xaf04, 0xffffffff, 0x00000100,
1031	0xae04, 0xffffffff, 0x00000100,
1032	0x949c, 0xffffffff, 0x00000100,
1033	0x802c, 0xffffffff, 0xe0000000,
1034	0x9160, 0xffffffff, 0x00010000,
1035	0x9164, 0xffffffff, 0x00030002,
1036	0x9168, 0xffffffff, 0x00040007,
1037	0x916c, 0xffffffff, 0x00060005,
1038	0x9170, 0xffffffff, 0x00090008,
1039	0x9174, 0xffffffff, 0x00020001,
1040	0x9178, 0xffffffff, 0x00040003,
1041	0x917c, 0xffffffff, 0x00000007,
1042	0x9180, 0xffffffff, 0x00060005,
1043	0x9184, 0xffffffff, 0x00090008,
1044	0x9188, 0xffffffff, 0x00030002,
1045	0x918c, 0xffffffff, 0x00050004,
1046	0x9190, 0xffffffff, 0x00000008,
1047	0x9194, 0xffffffff, 0x00070006,
1048	0x9198, 0xffffffff, 0x000a0009,
1049	0x919c, 0xffffffff, 0x00040003,
1050	0x91a0, 0xffffffff, 0x00060005,
1051	0x91a4, 0xffffffff, 0x00000009,
1052	0x91a8, 0xffffffff, 0x00080007,
1053	0x91ac, 0xffffffff, 0x000b000a,
1054	0x91b0, 0xffffffff, 0x00050004,
1055	0x91b4, 0xffffffff, 0x00070006,
1056	0x91b8, 0xffffffff, 0x0008000b,
1057	0x91bc, 0xffffffff, 0x000a0009,
1058	0x91c0, 0xffffffff, 0x000d000c,
1059	0x91c4, 0xffffffff, 0x00060005,
1060	0x91c8, 0xffffffff, 0x00080007,
1061	0x91cc, 0xffffffff, 0x0000000b,
1062	0x91d0, 0xffffffff, 0x000a0009,
1063	0x91d4, 0xffffffff, 0x000d000c,
1064	0x9150, 0xffffffff, 0x96940200,
1065	0x8708, 0xffffffff, 0x00900100,
1066	0xc478, 0xffffffff, 0x00000080,
1067	0xc404, 0xffffffff, 0x0020003f,
1068	0x30, 0xffffffff, 0x0000001c,
1069	0x34, 0x000f0000, 0x000f0000,
1070	0x160c, 0xffffffff, 0x00000100,
1071	0x1024, 0xffffffff, 0x00000100,
1072	0x20a8, 0xffffffff, 0x00000104,
1073	0x264c, 0x000c0000, 0x000c0000,
1074	0x2648, 0x000c0000, 0x000c0000,
1075	0x2f50, 0x00000001, 0x00000001,
1076	0x30cc, 0xc0000fff, 0x00000104,
1077	0xc1e4, 0x00000001, 0x00000001,
1078	0xd0c0, 0xfffffff0, 0x00000100,
1079	0xd8c0, 0xfffffff0, 0x00000100
1080};
1081
1082static u32 verde_pg_init[] = {
 
1083	0x353c, 0xffffffff, 0x40000,
1084	0x3538, 0xffffffff, 0x200010ff,
1085	0x353c, 0xffffffff, 0x0,
1086	0x353c, 0xffffffff, 0x0,
1087	0x353c, 0xffffffff, 0x0,
1088	0x353c, 0xffffffff, 0x0,
1089	0x353c, 0xffffffff, 0x0,
1090	0x353c, 0xffffffff, 0x7007,
1091	0x3538, 0xffffffff, 0x300010ff,
1092	0x353c, 0xffffffff, 0x0,
1093	0x353c, 0xffffffff, 0x0,
1094	0x353c, 0xffffffff, 0x0,
1095	0x353c, 0xffffffff, 0x0,
1096	0x353c, 0xffffffff, 0x0,
1097	0x353c, 0xffffffff, 0x400000,
1098	0x3538, 0xffffffff, 0x100010ff,
1099	0x353c, 0xffffffff, 0x0,
1100	0x353c, 0xffffffff, 0x0,
1101	0x353c, 0xffffffff, 0x0,
1102	0x353c, 0xffffffff, 0x0,
1103	0x353c, 0xffffffff, 0x0,
1104	0x353c, 0xffffffff, 0x120200,
1105	0x3538, 0xffffffff, 0x500010ff,
1106	0x353c, 0xffffffff, 0x0,
1107	0x353c, 0xffffffff, 0x0,
1108	0x353c, 0xffffffff, 0x0,
1109	0x353c, 0xffffffff, 0x0,
1110	0x353c, 0xffffffff, 0x0,
1111	0x353c, 0xffffffff, 0x1e1e16,
1112	0x3538, 0xffffffff, 0x600010ff,
1113	0x353c, 0xffffffff, 0x0,
1114	0x353c, 0xffffffff, 0x0,
1115	0x353c, 0xffffffff, 0x0,
1116	0x353c, 0xffffffff, 0x0,
1117	0x353c, 0xffffffff, 0x0,
1118	0x353c, 0xffffffff, 0x171f1e,
1119	0x3538, 0xffffffff, 0x700010ff,
1120	0x353c, 0xffffffff, 0x0,
1121	0x353c, 0xffffffff, 0x0,
1122	0x353c, 0xffffffff, 0x0,
1123	0x353c, 0xffffffff, 0x0,
1124	0x353c, 0xffffffff, 0x0,
1125	0x353c, 0xffffffff, 0x0,
1126	0x3538, 0xffffffff, 0x9ff,
1127	0x3500, 0xffffffff, 0x0,
1128	0x3504, 0xffffffff, 0x10000800,
1129	0x3504, 0xffffffff, 0xf,
1130	0x3504, 0xffffffff, 0xf,
1131	0x3500, 0xffffffff, 0x4,
1132	0x3504, 0xffffffff, 0x1000051e,
1133	0x3504, 0xffffffff, 0xffff,
1134	0x3504, 0xffffffff, 0xffff,
1135	0x3500, 0xffffffff, 0x8,
1136	0x3504, 0xffffffff, 0x80500,
1137	0x3500, 0xffffffff, 0x12,
1138	0x3504, 0xffffffff, 0x9050c,
1139	0x3500, 0xffffffff, 0x1d,
1140	0x3504, 0xffffffff, 0xb052c,
1141	0x3500, 0xffffffff, 0x2a,
1142	0x3504, 0xffffffff, 0x1053e,
1143	0x3500, 0xffffffff, 0x2d,
1144	0x3504, 0xffffffff, 0x10546,
1145	0x3500, 0xffffffff, 0x30,
1146	0x3504, 0xffffffff, 0xa054e,
1147	0x3500, 0xffffffff, 0x3c,
1148	0x3504, 0xffffffff, 0x1055f,
1149	0x3500, 0xffffffff, 0x3f,
1150	0x3504, 0xffffffff, 0x10567,
1151	0x3500, 0xffffffff, 0x42,
1152	0x3504, 0xffffffff, 0x1056f,
1153	0x3500, 0xffffffff, 0x45,
1154	0x3504, 0xffffffff, 0x10572,
1155	0x3500, 0xffffffff, 0x48,
1156	0x3504, 0xffffffff, 0x20575,
1157	0x3500, 0xffffffff, 0x4c,
1158	0x3504, 0xffffffff, 0x190801,
1159	0x3500, 0xffffffff, 0x67,
1160	0x3504, 0xffffffff, 0x1082a,
1161	0x3500, 0xffffffff, 0x6a,
1162	0x3504, 0xffffffff, 0x1b082d,
1163	0x3500, 0xffffffff, 0x87,
1164	0x3504, 0xffffffff, 0x310851,
1165	0x3500, 0xffffffff, 0xba,
1166	0x3504, 0xffffffff, 0x891,
1167	0x3500, 0xffffffff, 0xbc,
1168	0x3504, 0xffffffff, 0x893,
1169	0x3500, 0xffffffff, 0xbe,
1170	0x3504, 0xffffffff, 0x20895,
1171	0x3500, 0xffffffff, 0xc2,
1172	0x3504, 0xffffffff, 0x20899,
1173	0x3500, 0xffffffff, 0xc6,
1174	0x3504, 0xffffffff, 0x2089d,
1175	0x3500, 0xffffffff, 0xca,
1176	0x3504, 0xffffffff, 0x8a1,
1177	0x3500, 0xffffffff, 0xcc,
1178	0x3504, 0xffffffff, 0x8a3,
1179	0x3500, 0xffffffff, 0xce,
1180	0x3504, 0xffffffff, 0x308a5,
1181	0x3500, 0xffffffff, 0xd3,
1182	0x3504, 0xffffffff, 0x6d08cd,
1183	0x3500, 0xffffffff, 0x142,
1184	0x3504, 0xffffffff, 0x2000095a,
1185	0x3504, 0xffffffff, 0x1,
1186	0x3500, 0xffffffff, 0x144,
1187	0x3504, 0xffffffff, 0x301f095b,
1188	0x3500, 0xffffffff, 0x165,
1189	0x3504, 0xffffffff, 0xc094d,
1190	0x3500, 0xffffffff, 0x173,
1191	0x3504, 0xffffffff, 0xf096d,
1192	0x3500, 0xffffffff, 0x184,
1193	0x3504, 0xffffffff, 0x15097f,
1194	0x3500, 0xffffffff, 0x19b,
1195	0x3504, 0xffffffff, 0xc0998,
1196	0x3500, 0xffffffff, 0x1a9,
1197	0x3504, 0xffffffff, 0x409a7,
1198	0x3500, 0xffffffff, 0x1af,
1199	0x3504, 0xffffffff, 0xcdc,
1200	0x3500, 0xffffffff, 0x1b1,
1201	0x3504, 0xffffffff, 0x800,
1202	0x3508, 0xffffffff, 0x6c9b2000,
1203	0x3510, 0xfc00, 0x2000,
1204	0x3544, 0xffffffff, 0xfc0,
1205	0x28d4, 0x00000100, 0x100
1206};
1207
1208static void si_init_golden_registers(struct radeon_device *rdev)
1209{
1210	switch (rdev->family) {
1211	case CHIP_TAHITI:
1212		radeon_program_register_sequence(rdev,
1213						 tahiti_golden_registers,
1214						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1215		radeon_program_register_sequence(rdev,
1216						 tahiti_golden_rlc_registers,
1217						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1218		radeon_program_register_sequence(rdev,
1219						 tahiti_mgcg_cgcg_init,
1220						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1221		radeon_program_register_sequence(rdev,
1222						 tahiti_golden_registers2,
1223						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1224		break;
1225	case CHIP_PITCAIRN:
1226		radeon_program_register_sequence(rdev,
1227						 pitcairn_golden_registers,
1228						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1229		radeon_program_register_sequence(rdev,
1230						 pitcairn_golden_rlc_registers,
1231						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1232		radeon_program_register_sequence(rdev,
1233						 pitcairn_mgcg_cgcg_init,
1234						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1235		break;
1236	case CHIP_VERDE:
1237		radeon_program_register_sequence(rdev,
1238						 verde_golden_registers,
1239						 (const u32)ARRAY_SIZE(verde_golden_registers));
1240		radeon_program_register_sequence(rdev,
1241						 verde_golden_rlc_registers,
1242						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1243		radeon_program_register_sequence(rdev,
1244						 verde_mgcg_cgcg_init,
1245						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1246		radeon_program_register_sequence(rdev,
1247						 verde_pg_init,
1248						 (const u32)ARRAY_SIZE(verde_pg_init));
1249		break;
1250	case CHIP_OLAND:
1251		radeon_program_register_sequence(rdev,
1252						 oland_golden_registers,
1253						 (const u32)ARRAY_SIZE(oland_golden_registers));
1254		radeon_program_register_sequence(rdev,
1255						 oland_golden_rlc_registers,
1256						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1257		radeon_program_register_sequence(rdev,
1258						 oland_mgcg_cgcg_init,
1259						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1260		break;
1261	case CHIP_HAINAN:
1262		radeon_program_register_sequence(rdev,
1263						 hainan_golden_registers,
1264						 (const u32)ARRAY_SIZE(hainan_golden_registers));
1265		radeon_program_register_sequence(rdev,
1266						 hainan_golden_registers2,
1267						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1268		radeon_program_register_sequence(rdev,
1269						 hainan_mgcg_cgcg_init,
1270						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1271		break;
1272	default:
1273		break;
1274	}
1275}
1276
1277/**
1278 * si_get_allowed_info_register - fetch the register for the info ioctl
1279 *
1280 * @rdev: radeon_device pointer
1281 * @reg: register offset in bytes
1282 * @val: register value
1283 *
1284 * Returns 0 for success or -EINVAL for an invalid register
1285 *
1286 */
1287int si_get_allowed_info_register(struct radeon_device *rdev,
1288				 u32 reg, u32 *val)
1289{
1290	switch (reg) {
1291	case GRBM_STATUS:
1292	case GRBM_STATUS2:
1293	case GRBM_STATUS_SE0:
1294	case GRBM_STATUS_SE1:
1295	case SRBM_STATUS:
1296	case SRBM_STATUS2:
1297	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1298	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1299	case UVD_STATUS:
1300		*val = RREG32(reg);
1301		return 0;
1302	default:
1303		return -EINVAL;
1304	}
1305}
1306
1307#define PCIE_BUS_CLK                10000
1308#define TCLK                        (PCIE_BUS_CLK / 10)
1309
1310/**
1311 * si_get_xclk - get the xclk
1312 *
1313 * @rdev: radeon_device pointer
1314 *
1315 * Returns the reference clock used by the gfx engine
1316 * (SI).
1317 */
1318u32 si_get_xclk(struct radeon_device *rdev)
1319{
1320	u32 reference_clock = rdev->clock.spll.reference_freq;
1321	u32 tmp;
1322
1323	tmp = RREG32(CG_CLKPIN_CNTL_2);
1324	if (tmp & MUX_TCLK_TO_XCLK)
1325		return TCLK;
1326
1327	tmp = RREG32(CG_CLKPIN_CNTL);
1328	if (tmp & XTALIN_DIVIDE)
1329		return reference_clock / 4;
1330
1331	return reference_clock;
1332}
1333
1334/* get temperature in millidegrees */
1335int si_get_temp(struct radeon_device *rdev)
1336{
1337	u32 temp;
1338	int actual_temp = 0;
1339
1340	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1341		CTF_TEMP_SHIFT;
1342
1343	if (temp & 0x200)
1344		actual_temp = 255;
1345	else
1346		actual_temp = temp & 0x1ff;
1347
1348	actual_temp = (actual_temp * 1000);
1349
1350	return actual_temp;
1351}
1352
1353#define TAHITI_IO_MC_REGS_SIZE 36
1354
1355static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1356	{0x0000006f, 0x03044000},
1357	{0x00000070, 0x0480c018},
1358	{0x00000071, 0x00000040},
1359	{0x00000072, 0x01000000},
1360	{0x00000074, 0x000000ff},
1361	{0x00000075, 0x00143400},
1362	{0x00000076, 0x08ec0800},
1363	{0x00000077, 0x040000cc},
1364	{0x00000079, 0x00000000},
1365	{0x0000007a, 0x21000409},
1366	{0x0000007c, 0x00000000},
1367	{0x0000007d, 0xe8000000},
1368	{0x0000007e, 0x044408a8},
1369	{0x0000007f, 0x00000003},
1370	{0x00000080, 0x00000000},
1371	{0x00000081, 0x01000000},
1372	{0x00000082, 0x02000000},
1373	{0x00000083, 0x00000000},
1374	{0x00000084, 0xe3f3e4f4},
1375	{0x00000085, 0x00052024},
1376	{0x00000087, 0x00000000},
1377	{0x00000088, 0x66036603},
1378	{0x00000089, 0x01000000},
1379	{0x0000008b, 0x1c0a0000},
1380	{0x0000008c, 0xff010000},
1381	{0x0000008e, 0xffffefff},
1382	{0x0000008f, 0xfff3efff},
1383	{0x00000090, 0xfff3efbf},
1384	{0x00000094, 0x00101101},
1385	{0x00000095, 0x00000fff},
1386	{0x00000096, 0x00116fff},
1387	{0x00000097, 0x60010000},
1388	{0x00000098, 0x10010000},
1389	{0x00000099, 0x00006000},
1390	{0x0000009a, 0x00001000},
1391	{0x0000009f, 0x00a77400}
1392};
1393
1394static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1395	{0x0000006f, 0x03044000},
1396	{0x00000070, 0x0480c018},
1397	{0x00000071, 0x00000040},
1398	{0x00000072, 0x01000000},
1399	{0x00000074, 0x000000ff},
1400	{0x00000075, 0x00143400},
1401	{0x00000076, 0x08ec0800},
1402	{0x00000077, 0x040000cc},
1403	{0x00000079, 0x00000000},
1404	{0x0000007a, 0x21000409},
1405	{0x0000007c, 0x00000000},
1406	{0x0000007d, 0xe8000000},
1407	{0x0000007e, 0x044408a8},
1408	{0x0000007f, 0x00000003},
1409	{0x00000080, 0x00000000},
1410	{0x00000081, 0x01000000},
1411	{0x00000082, 0x02000000},
1412	{0x00000083, 0x00000000},
1413	{0x00000084, 0xe3f3e4f4},
1414	{0x00000085, 0x00052024},
1415	{0x00000087, 0x00000000},
1416	{0x00000088, 0x66036603},
1417	{0x00000089, 0x01000000},
1418	{0x0000008b, 0x1c0a0000},
1419	{0x0000008c, 0xff010000},
1420	{0x0000008e, 0xffffefff},
1421	{0x0000008f, 0xfff3efff},
1422	{0x00000090, 0xfff3efbf},
1423	{0x00000094, 0x00101101},
1424	{0x00000095, 0x00000fff},
1425	{0x00000096, 0x00116fff},
1426	{0x00000097, 0x60010000},
1427	{0x00000098, 0x10010000},
1428	{0x00000099, 0x00006000},
1429	{0x0000009a, 0x00001000},
1430	{0x0000009f, 0x00a47400}
1431};
1432
1433static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1434	{0x0000006f, 0x03044000},
1435	{0x00000070, 0x0480c018},
1436	{0x00000071, 0x00000040},
1437	{0x00000072, 0x01000000},
1438	{0x00000074, 0x000000ff},
1439	{0x00000075, 0x00143400},
1440	{0x00000076, 0x08ec0800},
1441	{0x00000077, 0x040000cc},
1442	{0x00000079, 0x00000000},
1443	{0x0000007a, 0x21000409},
1444	{0x0000007c, 0x00000000},
1445	{0x0000007d, 0xe8000000},
1446	{0x0000007e, 0x044408a8},
1447	{0x0000007f, 0x00000003},
1448	{0x00000080, 0x00000000},
1449	{0x00000081, 0x01000000},
1450	{0x00000082, 0x02000000},
1451	{0x00000083, 0x00000000},
1452	{0x00000084, 0xe3f3e4f4},
1453	{0x00000085, 0x00052024},
1454	{0x00000087, 0x00000000},
1455	{0x00000088, 0x66036603},
1456	{0x00000089, 0x01000000},
1457	{0x0000008b, 0x1c0a0000},
1458	{0x0000008c, 0xff010000},
1459	{0x0000008e, 0xffffefff},
1460	{0x0000008f, 0xfff3efff},
1461	{0x00000090, 0xfff3efbf},
1462	{0x00000094, 0x00101101},
1463	{0x00000095, 0x00000fff},
1464	{0x00000096, 0x00116fff},
1465	{0x00000097, 0x60010000},
1466	{0x00000098, 0x10010000},
1467	{0x00000099, 0x00006000},
1468	{0x0000009a, 0x00001000},
1469	{0x0000009f, 0x00a37400}
1470};
1471
1472static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1473	{0x0000006f, 0x03044000},
1474	{0x00000070, 0x0480c018},
1475	{0x00000071, 0x00000040},
1476	{0x00000072, 0x01000000},
1477	{0x00000074, 0x000000ff},
1478	{0x00000075, 0x00143400},
1479	{0x00000076, 0x08ec0800},
1480	{0x00000077, 0x040000cc},
1481	{0x00000079, 0x00000000},
1482	{0x0000007a, 0x21000409},
1483	{0x0000007c, 0x00000000},
1484	{0x0000007d, 0xe8000000},
1485	{0x0000007e, 0x044408a8},
1486	{0x0000007f, 0x00000003},
1487	{0x00000080, 0x00000000},
1488	{0x00000081, 0x01000000},
1489	{0x00000082, 0x02000000},
1490	{0x00000083, 0x00000000},
1491	{0x00000084, 0xe3f3e4f4},
1492	{0x00000085, 0x00052024},
1493	{0x00000087, 0x00000000},
1494	{0x00000088, 0x66036603},
1495	{0x00000089, 0x01000000},
1496	{0x0000008b, 0x1c0a0000},
1497	{0x0000008c, 0xff010000},
1498	{0x0000008e, 0xffffefff},
1499	{0x0000008f, 0xfff3efff},
1500	{0x00000090, 0xfff3efbf},
1501	{0x00000094, 0x00101101},
1502	{0x00000095, 0x00000fff},
1503	{0x00000096, 0x00116fff},
1504	{0x00000097, 0x60010000},
1505	{0x00000098, 0x10010000},
1506	{0x00000099, 0x00006000},
1507	{0x0000009a, 0x00001000},
1508	{0x0000009f, 0x00a17730}
1509};
1510
1511static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1512	{0x0000006f, 0x03044000},
1513	{0x00000070, 0x0480c018},
1514	{0x00000071, 0x00000040},
1515	{0x00000072, 0x01000000},
1516	{0x00000074, 0x000000ff},
1517	{0x00000075, 0x00143400},
1518	{0x00000076, 0x08ec0800},
1519	{0x00000077, 0x040000cc},
1520	{0x00000079, 0x00000000},
1521	{0x0000007a, 0x21000409},
1522	{0x0000007c, 0x00000000},
1523	{0x0000007d, 0xe8000000},
1524	{0x0000007e, 0x044408a8},
1525	{0x0000007f, 0x00000003},
1526	{0x00000080, 0x00000000},
1527	{0x00000081, 0x01000000},
1528	{0x00000082, 0x02000000},
1529	{0x00000083, 0x00000000},
1530	{0x00000084, 0xe3f3e4f4},
1531	{0x00000085, 0x00052024},
1532	{0x00000087, 0x00000000},
1533	{0x00000088, 0x66036603},
1534	{0x00000089, 0x01000000},
1535	{0x0000008b, 0x1c0a0000},
1536	{0x0000008c, 0xff010000},
1537	{0x0000008e, 0xffffefff},
1538	{0x0000008f, 0xfff3efff},
1539	{0x00000090, 0xfff3efbf},
1540	{0x00000094, 0x00101101},
1541	{0x00000095, 0x00000fff},
1542	{0x00000096, 0x00116fff},
1543	{0x00000097, 0x60010000},
1544	{0x00000098, 0x10010000},
1545	{0x00000099, 0x00006000},
1546	{0x0000009a, 0x00001000},
1547	{0x0000009f, 0x00a07730}
1548};
1549
1550/* ucode loading */
1551int si_mc_load_microcode(struct radeon_device *rdev)
1552{
1553	const __be32 *fw_data = NULL;
1554	const __le32 *new_fw_data = NULL;
1555	u32 running;
1556	u32 *io_mc_regs = NULL;
1557	const __le32 *new_io_mc_regs = NULL;
1558	int i, regs_size, ucode_size;
1559
1560	if (!rdev->mc_fw)
1561		return -EINVAL;
1562
1563	if (rdev->new_fw) {
1564		const struct mc_firmware_header_v1_0 *hdr =
1565			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1566
1567		radeon_ucode_print_mc_hdr(&hdr->header);
1568		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1569		new_io_mc_regs = (const __le32 *)
1570			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1571		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1572		new_fw_data = (const __le32 *)
1573			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1574	} else {
1575		ucode_size = rdev->mc_fw->size / 4;
1576
1577		switch (rdev->family) {
1578		case CHIP_TAHITI:
1579			io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1580			regs_size = TAHITI_IO_MC_REGS_SIZE;
1581			break;
1582		case CHIP_PITCAIRN:
1583			io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1584			regs_size = TAHITI_IO_MC_REGS_SIZE;
1585			break;
1586		case CHIP_VERDE:
1587		default:
1588			io_mc_regs = (u32 *)&verde_io_mc_regs;
1589			regs_size = TAHITI_IO_MC_REGS_SIZE;
1590			break;
1591		case CHIP_OLAND:
1592			io_mc_regs = (u32 *)&oland_io_mc_regs;
1593			regs_size = TAHITI_IO_MC_REGS_SIZE;
1594			break;
1595		case CHIP_HAINAN:
1596			io_mc_regs = (u32 *)&hainan_io_mc_regs;
1597			regs_size = TAHITI_IO_MC_REGS_SIZE;
1598			break;
1599		}
1600		fw_data = (const __be32 *)rdev->mc_fw->data;
1601	}
1602
1603	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1604
1605	if (running == 0) {
1606		/* reset the engine and set to writable */
1607		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1608		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1609
1610		/* load mc io regs */
1611		for (i = 0; i < regs_size; i++) {
1612			if (rdev->new_fw) {
1613				WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1614				WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1615			} else {
1616				WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1617				WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1618			}
1619		}
1620		/* load the MC ucode */
1621		for (i = 0; i < ucode_size; i++) {
1622			if (rdev->new_fw)
1623				WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1624			else
1625				WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1626		}
1627
1628		/* put the engine back into the active state */
1629		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1630		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1631		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1632
1633		/* wait for training to complete */
1634		for (i = 0; i < rdev->usec_timeout; i++) {
1635			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1636				break;
1637			udelay(1);
1638		}
1639		for (i = 0; i < rdev->usec_timeout; i++) {
1640			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1641				break;
1642			udelay(1);
1643		}
1644	}
1645
1646	return 0;
1647}
1648
1649static int si_init_microcode(struct radeon_device *rdev)
1650{
1651	const char *chip_name;
1652	const char *new_chip_name;
1653	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1654	size_t smc_req_size, mc2_req_size;
1655	char fw_name[30];
1656	int err;
1657	int new_fw = 0;
1658	bool new_smc = false;
1659	bool si58_fw = false;
1660	bool banks2_fw = false;
1661
1662	DRM_DEBUG("\n");
1663
1664	switch (rdev->family) {
1665	case CHIP_TAHITI:
1666		chip_name = "TAHITI";
1667		new_chip_name = "tahiti";
1668		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1669		me_req_size = SI_PM4_UCODE_SIZE * 4;
1670		ce_req_size = SI_CE_UCODE_SIZE * 4;
1671		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1672		mc_req_size = SI_MC_UCODE_SIZE * 4;
1673		mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1674		smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1675		break;
1676	case CHIP_PITCAIRN:
1677		chip_name = "PITCAIRN";
1678		if ((rdev->pdev->revision == 0x81) &&
1679		    ((rdev->pdev->device == 0x6810) ||
1680		     (rdev->pdev->device == 0x6811)))
1681			new_smc = true;
1682		new_chip_name = "pitcairn";
1683		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1684		me_req_size = SI_PM4_UCODE_SIZE * 4;
1685		ce_req_size = SI_CE_UCODE_SIZE * 4;
1686		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1687		mc_req_size = SI_MC_UCODE_SIZE * 4;
1688		mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1689		smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1690		break;
1691	case CHIP_VERDE:
1692		chip_name = "VERDE";
1693		if (((rdev->pdev->device == 0x6820) &&
1694		     ((rdev->pdev->revision == 0x81) ||
1695		      (rdev->pdev->revision == 0x83))) ||
1696		    ((rdev->pdev->device == 0x6821) &&
1697		     ((rdev->pdev->revision == 0x83) ||
1698		      (rdev->pdev->revision == 0x87))) ||
1699		    ((rdev->pdev->revision == 0x87) &&
1700		     ((rdev->pdev->device == 0x6823) ||
1701		      (rdev->pdev->device == 0x682b))))
1702			new_smc = true;
1703		new_chip_name = "verde";
1704		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1705		me_req_size = SI_PM4_UCODE_SIZE * 4;
1706		ce_req_size = SI_CE_UCODE_SIZE * 4;
1707		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1708		mc_req_size = SI_MC_UCODE_SIZE * 4;
1709		mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1710		smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1711		break;
1712	case CHIP_OLAND:
1713		chip_name = "OLAND";
1714		if (((rdev->pdev->revision == 0x81) &&
1715		     ((rdev->pdev->device == 0x6600) ||
1716		      (rdev->pdev->device == 0x6604) ||
1717		      (rdev->pdev->device == 0x6605) ||
1718		      (rdev->pdev->device == 0x6610))) ||
1719		    ((rdev->pdev->revision == 0x83) &&
1720		     (rdev->pdev->device == 0x6610)))
1721			new_smc = true;
1722		new_chip_name = "oland";
1723		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1724		me_req_size = SI_PM4_UCODE_SIZE * 4;
1725		ce_req_size = SI_CE_UCODE_SIZE * 4;
1726		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1727		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1728		smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1729		break;
1730	case CHIP_HAINAN:
1731		chip_name = "HAINAN";
1732		if (((rdev->pdev->revision == 0x81) &&
1733		     (rdev->pdev->device == 0x6660)) ||
1734		    ((rdev->pdev->revision == 0x83) &&
1735		     ((rdev->pdev->device == 0x6660) ||
1736		      (rdev->pdev->device == 0x6663) ||
1737		      (rdev->pdev->device == 0x6665) ||
1738		      (rdev->pdev->device == 0x6667))))
1739			new_smc = true;
1740		else if ((rdev->pdev->revision == 0xc3) &&
1741			 (rdev->pdev->device == 0x6665))
1742			banks2_fw = true;
1743		new_chip_name = "hainan";
1744		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1745		me_req_size = SI_PM4_UCODE_SIZE * 4;
1746		ce_req_size = SI_CE_UCODE_SIZE * 4;
1747		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1748		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1749		smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1750		break;
1751	default:
1752		BUG();
1753	}
1754
1755	/* this memory configuration requires special firmware */
1756	if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1757		si58_fw = true;
1758
1759	DRM_INFO("Loading %s Microcode\n", new_chip_name);
1760
1761	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1762	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1763	if (err) {
1764		snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1765		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1766		if (err)
1767			goto out;
1768		if (rdev->pfp_fw->size != pfp_req_size) {
1769			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1770			       rdev->pfp_fw->size, fw_name);
1771			err = -EINVAL;
1772			goto out;
1773		}
1774	} else {
1775		err = radeon_ucode_validate(rdev->pfp_fw);
1776		if (err) {
1777			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1778			       fw_name);
1779			goto out;
1780		} else {
1781			new_fw++;
1782		}
1783	}
1784
1785	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1786	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1787	if (err) {
1788		snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1789		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1790		if (err)
1791			goto out;
1792		if (rdev->me_fw->size != me_req_size) {
1793			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1794			       rdev->me_fw->size, fw_name);
1795			err = -EINVAL;
1796		}
1797	} else {
1798		err = radeon_ucode_validate(rdev->me_fw);
1799		if (err) {
1800			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1801			       fw_name);
1802			goto out;
1803		} else {
1804			new_fw++;
1805		}
1806	}
1807
1808	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1809	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1810	if (err) {
1811		snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1812		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1813		if (err)
1814			goto out;
1815		if (rdev->ce_fw->size != ce_req_size) {
1816			pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1817			       rdev->ce_fw->size, fw_name);
1818			err = -EINVAL;
1819		}
1820	} else {
1821		err = radeon_ucode_validate(rdev->ce_fw);
1822		if (err) {
1823			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1824			       fw_name);
1825			goto out;
1826		} else {
1827			new_fw++;
1828		}
1829	}
1830
1831	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1832	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1833	if (err) {
1834		snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1835		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1836		if (err)
1837			goto out;
1838		if (rdev->rlc_fw->size != rlc_req_size) {
1839			pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
1840			       rdev->rlc_fw->size, fw_name);
1841			err = -EINVAL;
1842		}
1843	} else {
1844		err = radeon_ucode_validate(rdev->rlc_fw);
1845		if (err) {
1846			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1847			       fw_name);
1848			goto out;
1849		} else {
1850			new_fw++;
1851		}
1852	}
1853
1854	if (si58_fw)
1855		snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1856	else
1857		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1858	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1859	if (err) {
1860		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1861		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1862		if (err) {
1863			snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1864			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1865			if (err)
1866				goto out;
1867		}
1868		if ((rdev->mc_fw->size != mc_req_size) &&
1869		    (rdev->mc_fw->size != mc2_req_size)) {
1870			pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
1871			       rdev->mc_fw->size, fw_name);
1872			err = -EINVAL;
1873		}
1874		DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1875	} else {
1876		err = radeon_ucode_validate(rdev->mc_fw);
1877		if (err) {
1878			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1879			       fw_name);
1880			goto out;
1881		} else {
1882			new_fw++;
1883		}
1884	}
1885
1886	if (banks2_fw)
1887		snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1888	else if (new_smc)
1889		snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1890	else
1891		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1892	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1893	if (err) {
1894		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1895		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1896		if (err) {
1897			pr_err("smc: error loading firmware \"%s\"\n", fw_name);
1898			release_firmware(rdev->smc_fw);
1899			rdev->smc_fw = NULL;
1900			err = 0;
1901		} else if (rdev->smc_fw->size != smc_req_size) {
1902			pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
1903			       rdev->smc_fw->size, fw_name);
1904			err = -EINVAL;
1905		}
1906	} else {
1907		err = radeon_ucode_validate(rdev->smc_fw);
1908		if (err) {
1909			pr_err("si_cp: validation failed for firmware \"%s\"\n",
1910			       fw_name);
1911			goto out;
1912		} else {
1913			new_fw++;
1914		}
1915	}
1916
1917	if (new_fw == 0) {
1918		rdev->new_fw = false;
1919	} else if (new_fw < 6) {
1920		pr_err("si_fw: mixing new and old firmware!\n");
1921		err = -EINVAL;
1922	} else {
1923		rdev->new_fw = true;
1924	}
1925out:
1926	if (err) {
1927		if (err != -EINVAL)
1928			pr_err("si_cp: Failed to load firmware \"%s\"\n",
1929			       fw_name);
1930		release_firmware(rdev->pfp_fw);
1931		rdev->pfp_fw = NULL;
1932		release_firmware(rdev->me_fw);
1933		rdev->me_fw = NULL;
1934		release_firmware(rdev->ce_fw);
1935		rdev->ce_fw = NULL;
1936		release_firmware(rdev->rlc_fw);
1937		rdev->rlc_fw = NULL;
1938		release_firmware(rdev->mc_fw);
1939		rdev->mc_fw = NULL;
1940		release_firmware(rdev->smc_fw);
1941		rdev->smc_fw = NULL;
1942	}
1943	return err;
1944}
1945
1946/* watermark setup */
1947static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1948				   struct radeon_crtc *radeon_crtc,
1949				   struct drm_display_mode *mode,
1950				   struct drm_display_mode *other_mode)
1951{
1952	u32 tmp, buffer_alloc, i;
1953	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1954	/*
1955	 * Line Buffer Setup
1956	 * There are 3 line buffers, each one shared by 2 display controllers.
1957	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1958	 * the display controllers.  The paritioning is done via one of four
1959	 * preset allocations specified in bits 21:20:
1960	 *  0 - half lb
1961	 *  2 - whole lb, other crtc must be disabled
1962	 */
1963	/* this can get tricky if we have two large displays on a paired group
1964	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1965	 * non-linked crtcs for maximum line buffer allocation.
1966	 */
1967	if (radeon_crtc->base.enabled && mode) {
1968		if (other_mode) {
1969			tmp = 0; /* 1/2 */
1970			buffer_alloc = 1;
1971		} else {
1972			tmp = 2; /* whole */
1973			buffer_alloc = 2;
1974		}
1975	} else {
1976		tmp = 0;
1977		buffer_alloc = 0;
1978	}
1979
1980	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1981	       DC_LB_MEMORY_CONFIG(tmp));
1982
1983	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1984	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1985	for (i = 0; i < rdev->usec_timeout; i++) {
1986		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1987		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1988			break;
1989		udelay(1);
1990	}
1991
1992	if (radeon_crtc->base.enabled && mode) {
1993		switch (tmp) {
1994		case 0:
1995		default:
1996			return 4096 * 2;
1997		case 2:
1998			return 8192 * 2;
1999		}
2000	}
2001
2002	/* controller not enabled, so no lb used */
2003	return 0;
2004}
2005
2006static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
2007{
2008	u32 tmp = RREG32(MC_SHARED_CHMAP);
2009
2010	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2011	case 0:
2012	default:
2013		return 1;
2014	case 1:
2015		return 2;
2016	case 2:
2017		return 4;
2018	case 3:
2019		return 8;
2020	case 4:
2021		return 3;
2022	case 5:
2023		return 6;
2024	case 6:
2025		return 10;
2026	case 7:
2027		return 12;
2028	case 8:
2029		return 16;
2030	}
2031}
2032
2033struct dce6_wm_params {
2034	u32 dram_channels; /* number of dram channels */
2035	u32 yclk;          /* bandwidth per dram data pin in kHz */
2036	u32 sclk;          /* engine clock in kHz */
2037	u32 disp_clk;      /* display clock in kHz */
2038	u32 src_width;     /* viewport width */
2039	u32 active_time;   /* active display time in ns */
2040	u32 blank_time;    /* blank time in ns */
2041	bool interlaced;    /* mode is interlaced */
2042	fixed20_12 vsc;    /* vertical scale ratio */
2043	u32 num_heads;     /* number of active crtcs */
2044	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2045	u32 lb_size;       /* line buffer allocated to pipe */
2046	u32 vtaps;         /* vertical scaler taps */
2047};
2048
2049static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2050{
2051	/* Calculate raw DRAM Bandwidth */
2052	fixed20_12 dram_efficiency; /* 0.7 */
2053	fixed20_12 yclk, dram_channels, bandwidth;
2054	fixed20_12 a;
2055
2056	a.full = dfixed_const(1000);
2057	yclk.full = dfixed_const(wm->yclk);
2058	yclk.full = dfixed_div(yclk, a);
2059	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2060	a.full = dfixed_const(10);
2061	dram_efficiency.full = dfixed_const(7);
2062	dram_efficiency.full = dfixed_div(dram_efficiency, a);
2063	bandwidth.full = dfixed_mul(dram_channels, yclk);
2064	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2065
2066	return dfixed_trunc(bandwidth);
2067}
2068
2069static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2070{
2071	/* Calculate DRAM Bandwidth and the part allocated to display. */
2072	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2073	fixed20_12 yclk, dram_channels, bandwidth;
2074	fixed20_12 a;
2075
2076	a.full = dfixed_const(1000);
2077	yclk.full = dfixed_const(wm->yclk);
2078	yclk.full = dfixed_div(yclk, a);
2079	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2080	a.full = dfixed_const(10);
2081	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2082	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2083	bandwidth.full = dfixed_mul(dram_channels, yclk);
2084	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2085
2086	return dfixed_trunc(bandwidth);
2087}
2088
2089static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2090{
2091	/* Calculate the display Data return Bandwidth */
2092	fixed20_12 return_efficiency; /* 0.8 */
2093	fixed20_12 sclk, bandwidth;
2094	fixed20_12 a;
2095
2096	a.full = dfixed_const(1000);
2097	sclk.full = dfixed_const(wm->sclk);
2098	sclk.full = dfixed_div(sclk, a);
2099	a.full = dfixed_const(10);
2100	return_efficiency.full = dfixed_const(8);
2101	return_efficiency.full = dfixed_div(return_efficiency, a);
2102	a.full = dfixed_const(32);
2103	bandwidth.full = dfixed_mul(a, sclk);
2104	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2105
2106	return dfixed_trunc(bandwidth);
2107}
2108
2109static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2110{
2111	return 32;
2112}
2113
2114static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2115{
2116	/* Calculate the DMIF Request Bandwidth */
2117	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2118	fixed20_12 disp_clk, sclk, bandwidth;
2119	fixed20_12 a, b1, b2;
2120	u32 min_bandwidth;
2121
2122	a.full = dfixed_const(1000);
2123	disp_clk.full = dfixed_const(wm->disp_clk);
2124	disp_clk.full = dfixed_div(disp_clk, a);
2125	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2126	b1.full = dfixed_mul(a, disp_clk);
2127
2128	a.full = dfixed_const(1000);
2129	sclk.full = dfixed_const(wm->sclk);
2130	sclk.full = dfixed_div(sclk, a);
2131	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2132	b2.full = dfixed_mul(a, sclk);
2133
2134	a.full = dfixed_const(10);
2135	disp_clk_request_efficiency.full = dfixed_const(8);
2136	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2137
2138	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2139
2140	a.full = dfixed_const(min_bandwidth);
2141	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2142
2143	return dfixed_trunc(bandwidth);
2144}
2145
2146static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2147{
2148	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2149	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2150	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2151	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2152
2153	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2154}
2155
2156static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2157{
2158	/* Calculate the display mode Average Bandwidth
2159	 * DisplayMode should contain the source and destination dimensions,
2160	 * timing, etc.
2161	 */
2162	fixed20_12 bpp;
2163	fixed20_12 line_time;
2164	fixed20_12 src_width;
2165	fixed20_12 bandwidth;
2166	fixed20_12 a;
2167
2168	a.full = dfixed_const(1000);
2169	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2170	line_time.full = dfixed_div(line_time, a);
2171	bpp.full = dfixed_const(wm->bytes_per_pixel);
2172	src_width.full = dfixed_const(wm->src_width);
2173	bandwidth.full = dfixed_mul(src_width, bpp);
2174	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2175	bandwidth.full = dfixed_div(bandwidth, line_time);
2176
2177	return dfixed_trunc(bandwidth);
2178}
2179
2180static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2181{
2182	/* First calcualte the latency in ns */
2183	u32 mc_latency = 2000; /* 2000 ns. */
2184	u32 available_bandwidth = dce6_available_bandwidth(wm);
2185	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2186	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2187	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2188	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2189		(wm->num_heads * cursor_line_pair_return_time);
2190	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2191	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2192	u32 tmp, dmif_size = 12288;
2193	fixed20_12 a, b, c;
2194
2195	if (wm->num_heads == 0)
2196		return 0;
2197
2198	a.full = dfixed_const(2);
2199	b.full = dfixed_const(1);
2200	if ((wm->vsc.full > a.full) ||
2201	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2202	    (wm->vtaps >= 5) ||
2203	    ((wm->vsc.full >= a.full) && wm->interlaced))
2204		max_src_lines_per_dst_line = 4;
2205	else
2206		max_src_lines_per_dst_line = 2;
2207
2208	a.full = dfixed_const(available_bandwidth);
2209	b.full = dfixed_const(wm->num_heads);
2210	a.full = dfixed_div(a, b);
2211	tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2212	tmp = min(dfixed_trunc(a), tmp);
2213
2214	lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
2215
2216	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2217	b.full = dfixed_const(1000);
2218	c.full = dfixed_const(lb_fill_bw);
2219	b.full = dfixed_div(c, b);
2220	a.full = dfixed_div(a, b);
2221	line_fill_time = dfixed_trunc(a);
2222
2223	if (line_fill_time < wm->active_time)
2224		return latency;
2225	else
2226		return latency + (line_fill_time - wm->active_time);
2227
2228}
2229
2230static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2231{
2232	if (dce6_average_bandwidth(wm) <=
2233	    (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2234		return true;
2235	else
2236		return false;
2237};
2238
2239static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2240{
2241	if (dce6_average_bandwidth(wm) <=
2242	    (dce6_available_bandwidth(wm) / wm->num_heads))
2243		return true;
2244	else
2245		return false;
2246};
2247
2248static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2249{
2250	u32 lb_partitions = wm->lb_size / wm->src_width;
2251	u32 line_time = wm->active_time + wm->blank_time;
2252	u32 latency_tolerant_lines;
2253	u32 latency_hiding;
2254	fixed20_12 a;
2255
2256	a.full = dfixed_const(1);
2257	if (wm->vsc.full > a.full)
2258		latency_tolerant_lines = 1;
2259	else {
2260		if (lb_partitions <= (wm->vtaps + 1))
2261			latency_tolerant_lines = 1;
2262		else
2263			latency_tolerant_lines = 2;
2264	}
2265
2266	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2267
2268	if (dce6_latency_watermark(wm) <= latency_hiding)
2269		return true;
2270	else
2271		return false;
2272}
2273
2274static void dce6_program_watermarks(struct radeon_device *rdev,
2275					 struct radeon_crtc *radeon_crtc,
2276					 u32 lb_size, u32 num_heads)
2277{
2278	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2279	struct dce6_wm_params wm_low, wm_high;
2280	u32 dram_channels;
2281	u32 active_time;
2282	u32 line_time = 0;
2283	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2284	u32 priority_a_mark = 0, priority_b_mark = 0;
2285	u32 priority_a_cnt = PRIORITY_OFF;
2286	u32 priority_b_cnt = PRIORITY_OFF;
2287	u32 tmp, arb_control3;
2288	fixed20_12 a, b, c;
2289
2290	if (radeon_crtc->base.enabled && num_heads && mode) {
2291		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2292					    (u32)mode->clock);
2293		line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2294					  (u32)mode->clock);
2295		line_time = min(line_time, (u32)65535);
2296		priority_a_cnt = 0;
2297		priority_b_cnt = 0;
2298
2299		if (rdev->family == CHIP_ARUBA)
2300			dram_channels = evergreen_get_number_of_dram_channels(rdev);
2301		else
2302			dram_channels = si_get_number_of_dram_channels(rdev);
2303
2304		/* watermark for high clocks */
2305		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2306			wm_high.yclk =
2307				radeon_dpm_get_mclk(rdev, false) * 10;
2308			wm_high.sclk =
2309				radeon_dpm_get_sclk(rdev, false) * 10;
2310		} else {
2311			wm_high.yclk = rdev->pm.current_mclk * 10;
2312			wm_high.sclk = rdev->pm.current_sclk * 10;
2313		}
2314
2315		wm_high.disp_clk = mode->clock;
2316		wm_high.src_width = mode->crtc_hdisplay;
2317		wm_high.active_time = active_time;
2318		wm_high.blank_time = line_time - wm_high.active_time;
2319		wm_high.interlaced = false;
2320		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2321			wm_high.interlaced = true;
2322		wm_high.vsc = radeon_crtc->vsc;
2323		wm_high.vtaps = 1;
2324		if (radeon_crtc->rmx_type != RMX_OFF)
2325			wm_high.vtaps = 2;
2326		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2327		wm_high.lb_size = lb_size;
2328		wm_high.dram_channels = dram_channels;
2329		wm_high.num_heads = num_heads;
2330
2331		/* watermark for low clocks */
2332		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2333			wm_low.yclk =
2334				radeon_dpm_get_mclk(rdev, true) * 10;
2335			wm_low.sclk =
2336				radeon_dpm_get_sclk(rdev, true) * 10;
2337		} else {
2338			wm_low.yclk = rdev->pm.current_mclk * 10;
2339			wm_low.sclk = rdev->pm.current_sclk * 10;
2340		}
2341
2342		wm_low.disp_clk = mode->clock;
2343		wm_low.src_width = mode->crtc_hdisplay;
2344		wm_low.active_time = active_time;
2345		wm_low.blank_time = line_time - wm_low.active_time;
2346		wm_low.interlaced = false;
2347		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2348			wm_low.interlaced = true;
2349		wm_low.vsc = radeon_crtc->vsc;
2350		wm_low.vtaps = 1;
2351		if (radeon_crtc->rmx_type != RMX_OFF)
2352			wm_low.vtaps = 2;
2353		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2354		wm_low.lb_size = lb_size;
2355		wm_low.dram_channels = dram_channels;
2356		wm_low.num_heads = num_heads;
2357
2358		/* set for high clocks */
2359		latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2360		/* set for low clocks */
2361		latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2362
2363		/* possibly force display priority to high */
2364		/* should really do this at mode validation time... */
2365		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2366		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2367		    !dce6_check_latency_hiding(&wm_high) ||
2368		    (rdev->disp_priority == 2)) {
2369			DRM_DEBUG_KMS("force priority to high\n");
2370			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2371			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2372		}
2373		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2374		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2375		    !dce6_check_latency_hiding(&wm_low) ||
2376		    (rdev->disp_priority == 2)) {
2377			DRM_DEBUG_KMS("force priority to high\n");
2378			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2379			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2380		}
2381
2382		a.full = dfixed_const(1000);
2383		b.full = dfixed_const(mode->clock);
2384		b.full = dfixed_div(b, a);
2385		c.full = dfixed_const(latency_watermark_a);
2386		c.full = dfixed_mul(c, b);
2387		c.full = dfixed_mul(c, radeon_crtc->hsc);
2388		c.full = dfixed_div(c, a);
2389		a.full = dfixed_const(16);
2390		c.full = dfixed_div(c, a);
2391		priority_a_mark = dfixed_trunc(c);
2392		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2393
2394		a.full = dfixed_const(1000);
2395		b.full = dfixed_const(mode->clock);
2396		b.full = dfixed_div(b, a);
2397		c.full = dfixed_const(latency_watermark_b);
2398		c.full = dfixed_mul(c, b);
2399		c.full = dfixed_mul(c, radeon_crtc->hsc);
2400		c.full = dfixed_div(c, a);
2401		a.full = dfixed_const(16);
2402		c.full = dfixed_div(c, a);
2403		priority_b_mark = dfixed_trunc(c);
2404		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2405
2406		/* Save number of lines the linebuffer leads before the scanout */
2407		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2408	}
2409
2410	/* select wm A */
2411	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2412	tmp = arb_control3;
2413	tmp &= ~LATENCY_WATERMARK_MASK(3);
2414	tmp |= LATENCY_WATERMARK_MASK(1);
2415	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2416	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2417	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2418		LATENCY_HIGH_WATERMARK(line_time)));
2419	/* select wm B */
2420	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2421	tmp &= ~LATENCY_WATERMARK_MASK(3);
2422	tmp |= LATENCY_WATERMARK_MASK(2);
2423	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2424	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2425	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2426		LATENCY_HIGH_WATERMARK(line_time)));
2427	/* restore original selection */
2428	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2429
2430	/* write the priority marks */
2431	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2432	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2433
2434	/* save values for DPM */
2435	radeon_crtc->line_time = line_time;
2436	radeon_crtc->wm_high = latency_watermark_a;
2437	radeon_crtc->wm_low = latency_watermark_b;
2438}
2439
2440void dce6_bandwidth_update(struct radeon_device *rdev)
2441{
2442	struct drm_display_mode *mode0 = NULL;
2443	struct drm_display_mode *mode1 = NULL;
2444	u32 num_heads = 0, lb_size;
2445	int i;
2446
2447	if (!rdev->mode_info.mode_config_initialized)
2448		return;
2449
2450	radeon_update_display_priority(rdev);
2451
2452	for (i = 0; i < rdev->num_crtc; i++) {
2453		if (rdev->mode_info.crtcs[i]->base.enabled)
2454			num_heads++;
2455	}
2456	for (i = 0; i < rdev->num_crtc; i += 2) {
2457		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2458		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2459		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2460		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2461		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2462		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2463	}
2464}
2465
2466/*
2467 * Core functions
2468 */
2469static void si_tiling_mode_table_init(struct radeon_device *rdev)
2470{
2471	u32 *tile = rdev->config.si.tile_mode_array;
2472	const u32 num_tile_mode_states =
2473			ARRAY_SIZE(rdev->config.si.tile_mode_array);
2474	u32 reg_offset, split_equal_to_row_size;
2475
2476	switch (rdev->config.si.mem_row_size_in_kb) {
2477	case 1:
2478		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2479		break;
2480	case 2:
2481	default:
2482		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2483		break;
2484	case 4:
2485		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2486		break;
2487	}
2488
2489	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2490		tile[reg_offset] = 0;
2491
2492	switch(rdev->family) {
2493	case CHIP_TAHITI:
2494	case CHIP_PITCAIRN:
2495		/* non-AA compressed depth or any compressed stencil */
2496		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2500			   NUM_BANKS(ADDR_SURF_16_BANK) |
2501			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2504		/* 2xAA/4xAA compressed depth only */
2505		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2508			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2509			   NUM_BANKS(ADDR_SURF_16_BANK) |
2510			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2513		/* 8xAA compressed depth only */
2514		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2516			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2517			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2518			   NUM_BANKS(ADDR_SURF_16_BANK) |
2519			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2521			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2522		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2523		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2527			   NUM_BANKS(ADDR_SURF_16_BANK) |
2528			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2531		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2532		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2536			   NUM_BANKS(ADDR_SURF_16_BANK) |
2537			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2539			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2541		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2544			   TILE_SPLIT(split_equal_to_row_size) |
2545			   NUM_BANKS(ADDR_SURF_16_BANK) |
2546			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2550		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2551			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2553			   TILE_SPLIT(split_equal_to_row_size) |
2554			   NUM_BANKS(ADDR_SURF_16_BANK) |
2555			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2557			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2558		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2559		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2560			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2561			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562			   TILE_SPLIT(split_equal_to_row_size) |
2563			   NUM_BANKS(ADDR_SURF_16_BANK) |
2564			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2566			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2567		/* 1D and 1D Array Surfaces */
2568		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2569			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2570			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2572			   NUM_BANKS(ADDR_SURF_16_BANK) |
2573			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2575			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2576		/* Displayable maps. */
2577		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2578			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2579			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2580			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2581			   NUM_BANKS(ADDR_SURF_16_BANK) |
2582			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2584			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2585		/* Display 8bpp. */
2586		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2590			   NUM_BANKS(ADDR_SURF_16_BANK) |
2591			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2593			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594		/* Display 16bpp. */
2595		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599			   NUM_BANKS(ADDR_SURF_16_BANK) |
2600			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603		/* Display 32bpp. */
2604		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2606			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2607			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2608			   NUM_BANKS(ADDR_SURF_16_BANK) |
2609			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2610			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2611			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2612		/* Thin. */
2613		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2617			   NUM_BANKS(ADDR_SURF_16_BANK) |
2618			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2621		/* Thin 8 bpp. */
2622		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626			   NUM_BANKS(ADDR_SURF_16_BANK) |
2627			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2629			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2630		/* Thin 16 bpp. */
2631		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2634			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2635			   NUM_BANKS(ADDR_SURF_16_BANK) |
2636			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2639		/* Thin 32 bpp. */
2640		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2643			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2644			   NUM_BANKS(ADDR_SURF_16_BANK) |
2645			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2647			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2648		/* Thin 64 bpp. */
2649		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652			   TILE_SPLIT(split_equal_to_row_size) |
2653			   NUM_BANKS(ADDR_SURF_16_BANK) |
2654			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2657		/* 8 bpp PRT. */
2658		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2659			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2660			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2661			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2662			   NUM_BANKS(ADDR_SURF_16_BANK) |
2663			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2664			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2665			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2666		/* 16 bpp PRT */
2667		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2669			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2670			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2671			   NUM_BANKS(ADDR_SURF_16_BANK) |
2672			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2674			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2675		/* 32 bpp PRT */
2676		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2680			   NUM_BANKS(ADDR_SURF_16_BANK) |
2681			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2683			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2684		/* 64 bpp PRT */
2685		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2689			   NUM_BANKS(ADDR_SURF_16_BANK) |
2690			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2693		/* 128 bpp PRT */
2694		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2696			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2697			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2698			   NUM_BANKS(ADDR_SURF_8_BANK) |
2699			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2701			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2702
2703		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2704			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2705		break;
2706
2707	case CHIP_VERDE:
2708	case CHIP_OLAND:
2709	case CHIP_HAINAN:
2710		/* non-AA compressed depth or any compressed stencil */
2711		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2715			   NUM_BANKS(ADDR_SURF_16_BANK) |
2716			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2719		/* 2xAA/4xAA compressed depth only */
2720		tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2721			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2722			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2724			   NUM_BANKS(ADDR_SURF_16_BANK) |
2725			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2727			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2728		/* 8xAA compressed depth only */
2729		tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2730			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2731			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2733			   NUM_BANKS(ADDR_SURF_16_BANK) |
2734			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2736			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2737		/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2738		tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2740			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2742			   NUM_BANKS(ADDR_SURF_16_BANK) |
2743			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746		/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2747		tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2748			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2749			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2751			   NUM_BANKS(ADDR_SURF_16_BANK) |
2752			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2754			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2755		/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2756		tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2758			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759			   TILE_SPLIT(split_equal_to_row_size) |
2760			   NUM_BANKS(ADDR_SURF_16_BANK) |
2761			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2763			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2764		/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2765		tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768			   TILE_SPLIT(split_equal_to_row_size) |
2769			   NUM_BANKS(ADDR_SURF_16_BANK) |
2770			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2772			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773		/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2774		tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775			   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2776			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777			   TILE_SPLIT(split_equal_to_row_size) |
2778			   NUM_BANKS(ADDR_SURF_16_BANK) |
2779			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2781			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2782		/* 1D and 1D Array Surfaces */
2783		tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2784			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2785			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2787			   NUM_BANKS(ADDR_SURF_16_BANK) |
2788			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2789			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2790			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2791		/* Displayable maps. */
2792		tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2793			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2794			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2796			   NUM_BANKS(ADDR_SURF_16_BANK) |
2797			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2799			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2800		/* Display 8bpp. */
2801		tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2802			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2803			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2805			   NUM_BANKS(ADDR_SURF_16_BANK) |
2806			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2808			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2809		/* Display 16bpp. */
2810		tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2811			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2812			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2814			   NUM_BANKS(ADDR_SURF_16_BANK) |
2815			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2817			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2818		/* Display 32bpp. */
2819		tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820			   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2823			   NUM_BANKS(ADDR_SURF_16_BANK) |
2824			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2827		/* Thin. */
2828		tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2829			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2830			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2831			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2832			   NUM_BANKS(ADDR_SURF_16_BANK) |
2833			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2835			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2836		/* Thin 8 bpp. */
2837		tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2839			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2840			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2841			   NUM_BANKS(ADDR_SURF_16_BANK) |
2842			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2844			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2845		/* Thin 16 bpp. */
2846		tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2847			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2848			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2849			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2850			   NUM_BANKS(ADDR_SURF_16_BANK) |
2851			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2853			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2854		/* Thin 32 bpp. */
2855		tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2859			   NUM_BANKS(ADDR_SURF_16_BANK) |
2860			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2863		/* Thin 64 bpp. */
2864		tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2865			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2866			   PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2867			   TILE_SPLIT(split_equal_to_row_size) |
2868			   NUM_BANKS(ADDR_SURF_16_BANK) |
2869			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2871			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2872		/* 8 bpp PRT. */
2873		tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2875			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2876			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2877			   NUM_BANKS(ADDR_SURF_16_BANK) |
2878			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2879			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2880			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2881		/* 16 bpp PRT */
2882		tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2883			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2884			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2885			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2886			   NUM_BANKS(ADDR_SURF_16_BANK) |
2887			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2888			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2889			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2890		/* 32 bpp PRT */
2891		tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2895			   NUM_BANKS(ADDR_SURF_16_BANK) |
2896			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2898			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2899		/* 64 bpp PRT */
2900		tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2902			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2903			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2904			   NUM_BANKS(ADDR_SURF_16_BANK) |
2905			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2906			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2907			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2908		/* 128 bpp PRT */
2909		tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910			   MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2911			   PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2912			   TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2913			   NUM_BANKS(ADDR_SURF_8_BANK) |
2914			   BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2915			   BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2916			   MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2917
2918		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2919			WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2920		break;
2921
2922	default:
2923		DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2924	}
2925}
2926
2927static void si_select_se_sh(struct radeon_device *rdev,
2928			    u32 se_num, u32 sh_num)
2929{
2930	u32 data = INSTANCE_BROADCAST_WRITES;
2931
2932	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2933		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2934	else if (se_num == 0xffffffff)
2935		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2936	else if (sh_num == 0xffffffff)
2937		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2938	else
2939		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2940	WREG32(GRBM_GFX_INDEX, data);
2941}
2942
2943static u32 si_create_bitmask(u32 bit_width)
2944{
2945	u32 i, mask = 0;
2946
2947	for (i = 0; i < bit_width; i++) {
2948		mask <<= 1;
2949		mask |= 1;
2950	}
2951	return mask;
2952}
2953
2954static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2955{
2956	u32 data, mask;
2957
2958	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2959	if (data & 1)
2960		data &= INACTIVE_CUS_MASK;
2961	else
2962		data = 0;
2963	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2964
2965	data >>= INACTIVE_CUS_SHIFT;
2966
2967	mask = si_create_bitmask(cu_per_sh);
2968
2969	return ~data & mask;
2970}
2971
2972static void si_setup_spi(struct radeon_device *rdev,
2973			 u32 se_num, u32 sh_per_se,
2974			 u32 cu_per_sh)
2975{
2976	int i, j, k;
2977	u32 data, mask, active_cu;
2978
2979	for (i = 0; i < se_num; i++) {
2980		for (j = 0; j < sh_per_se; j++) {
2981			si_select_se_sh(rdev, i, j);
2982			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2983			active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2984
2985			mask = 1;
2986			for (k = 0; k < 16; k++) {
2987				mask <<= k;
2988				if (active_cu & mask) {
2989					data &= ~mask;
2990					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2991					break;
2992				}
2993			}
2994		}
2995	}
2996	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2997}
2998
2999static u32 si_get_rb_disabled(struct radeon_device *rdev,
3000			      u32 max_rb_num_per_se,
3001			      u32 sh_per_se)
3002{
3003	u32 data, mask;
3004
3005	data = RREG32(CC_RB_BACKEND_DISABLE);
3006	if (data & 1)
3007		data &= BACKEND_DISABLE_MASK;
3008	else
3009		data = 0;
3010	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3011
3012	data >>= BACKEND_DISABLE_SHIFT;
3013
3014	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3015
3016	return data & mask;
3017}
3018
3019static void si_setup_rb(struct radeon_device *rdev,
3020			u32 se_num, u32 sh_per_se,
3021			u32 max_rb_num_per_se)
3022{
3023	int i, j;
3024	u32 data, mask;
3025	u32 disabled_rbs = 0;
3026	u32 enabled_rbs = 0;
3027
3028	for (i = 0; i < se_num; i++) {
3029		for (j = 0; j < sh_per_se; j++) {
3030			si_select_se_sh(rdev, i, j);
3031			data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3032			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3033		}
3034	}
3035	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3036
3037	mask = 1;
3038	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3039		if (!(disabled_rbs & mask))
3040			enabled_rbs |= mask;
3041		mask <<= 1;
3042	}
3043
3044	rdev->config.si.backend_enable_mask = enabled_rbs;
3045
3046	for (i = 0; i < se_num; i++) {
3047		si_select_se_sh(rdev, i, 0xffffffff);
3048		data = 0;
3049		for (j = 0; j < sh_per_se; j++) {
3050			switch (enabled_rbs & 3) {
3051			case 1:
3052				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3053				break;
3054			case 2:
3055				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3056				break;
3057			case 3:
3058			default:
3059				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3060				break;
3061			}
3062			enabled_rbs >>= 2;
3063		}
3064		WREG32(PA_SC_RASTER_CONFIG, data);
3065	}
3066	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3067}
3068
3069static void si_gpu_init(struct radeon_device *rdev)
3070{
3071	u32 gb_addr_config = 0;
3072	u32 mc_arb_ramcfg;
3073	u32 sx_debug_1;
3074	u32 hdp_host_path_cntl;
3075	u32 tmp;
3076	int i, j;
3077
3078	switch (rdev->family) {
3079	case CHIP_TAHITI:
3080		rdev->config.si.max_shader_engines = 2;
3081		rdev->config.si.max_tile_pipes = 12;
3082		rdev->config.si.max_cu_per_sh = 8;
3083		rdev->config.si.max_sh_per_se = 2;
3084		rdev->config.si.max_backends_per_se = 4;
3085		rdev->config.si.max_texture_channel_caches = 12;
3086		rdev->config.si.max_gprs = 256;
3087		rdev->config.si.max_gs_threads = 32;
3088		rdev->config.si.max_hw_contexts = 8;
3089
3090		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3091		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3092		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3093		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3094		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3095		break;
3096	case CHIP_PITCAIRN:
3097		rdev->config.si.max_shader_engines = 2;
3098		rdev->config.si.max_tile_pipes = 8;
3099		rdev->config.si.max_cu_per_sh = 5;
3100		rdev->config.si.max_sh_per_se = 2;
3101		rdev->config.si.max_backends_per_se = 4;
3102		rdev->config.si.max_texture_channel_caches = 8;
3103		rdev->config.si.max_gprs = 256;
3104		rdev->config.si.max_gs_threads = 32;
3105		rdev->config.si.max_hw_contexts = 8;
3106
3107		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3108		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3109		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3110		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3111		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3112		break;
3113	case CHIP_VERDE:
3114	default:
3115		rdev->config.si.max_shader_engines = 1;
3116		rdev->config.si.max_tile_pipes = 4;
3117		rdev->config.si.max_cu_per_sh = 5;
3118		rdev->config.si.max_sh_per_se = 2;
3119		rdev->config.si.max_backends_per_se = 4;
3120		rdev->config.si.max_texture_channel_caches = 4;
3121		rdev->config.si.max_gprs = 256;
3122		rdev->config.si.max_gs_threads = 32;
3123		rdev->config.si.max_hw_contexts = 8;
3124
3125		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3127		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3130		break;
3131	case CHIP_OLAND:
3132		rdev->config.si.max_shader_engines = 1;
3133		rdev->config.si.max_tile_pipes = 4;
3134		rdev->config.si.max_cu_per_sh = 6;
3135		rdev->config.si.max_sh_per_se = 1;
3136		rdev->config.si.max_backends_per_se = 2;
3137		rdev->config.si.max_texture_channel_caches = 4;
3138		rdev->config.si.max_gprs = 256;
3139		rdev->config.si.max_gs_threads = 16;
3140		rdev->config.si.max_hw_contexts = 8;
3141
3142		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3143		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3144		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3145		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3146		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3147		break;
3148	case CHIP_HAINAN:
3149		rdev->config.si.max_shader_engines = 1;
3150		rdev->config.si.max_tile_pipes = 4;
3151		rdev->config.si.max_cu_per_sh = 5;
3152		rdev->config.si.max_sh_per_se = 1;
3153		rdev->config.si.max_backends_per_se = 1;
3154		rdev->config.si.max_texture_channel_caches = 2;
3155		rdev->config.si.max_gprs = 256;
3156		rdev->config.si.max_gs_threads = 16;
3157		rdev->config.si.max_hw_contexts = 8;
3158
3159		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3160		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3161		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3162		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3163		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3164		break;
3165	}
3166
3167	/* Initialize HDP */
3168	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3169		WREG32((0x2c14 + j), 0x00000000);
3170		WREG32((0x2c18 + j), 0x00000000);
3171		WREG32((0x2c1c + j), 0x00000000);
3172		WREG32((0x2c20 + j), 0x00000000);
3173		WREG32((0x2c24 + j), 0x00000000);
3174	}
3175
3176	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3177	WREG32(SRBM_INT_CNTL, 1);
3178	WREG32(SRBM_INT_ACK, 1);
3179
3180	evergreen_fix_pci_max_read_req_size(rdev);
3181
3182	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3183
3184	RREG32(MC_SHARED_CHMAP);
3185	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3186
3187	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3188	rdev->config.si.mem_max_burst_length_bytes = 256;
3189	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3190	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3191	if (rdev->config.si.mem_row_size_in_kb > 4)
3192		rdev->config.si.mem_row_size_in_kb = 4;
3193	/* XXX use MC settings? */
3194	rdev->config.si.shader_engine_tile_size = 32;
3195	rdev->config.si.num_gpus = 1;
3196	rdev->config.si.multi_gpu_tile_size = 64;
3197
3198	/* fix up row size */
3199	gb_addr_config &= ~ROW_SIZE_MASK;
3200	switch (rdev->config.si.mem_row_size_in_kb) {
3201	case 1:
3202	default:
3203		gb_addr_config |= ROW_SIZE(0);
3204		break;
3205	case 2:
3206		gb_addr_config |= ROW_SIZE(1);
3207		break;
3208	case 4:
3209		gb_addr_config |= ROW_SIZE(2);
3210		break;
3211	}
3212
3213	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3214	 * not have bank info, so create a custom tiling dword.
3215	 * bits 3:0   num_pipes
3216	 * bits 7:4   num_banks
3217	 * bits 11:8  group_size
3218	 * bits 15:12 row_size
3219	 */
3220	rdev->config.si.tile_config = 0;
3221	switch (rdev->config.si.num_tile_pipes) {
3222	case 1:
3223		rdev->config.si.tile_config |= (0 << 0);
3224		break;
3225	case 2:
3226		rdev->config.si.tile_config |= (1 << 0);
3227		break;
3228	case 4:
3229		rdev->config.si.tile_config |= (2 << 0);
3230		break;
3231	case 8:
3232	default:
3233		/* XXX what about 12? */
3234		rdev->config.si.tile_config |= (3 << 0);
3235		break;
3236	}
3237	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3238	case 0: /* four banks */
3239		rdev->config.si.tile_config |= 0 << 4;
3240		break;
3241	case 1: /* eight banks */
3242		rdev->config.si.tile_config |= 1 << 4;
3243		break;
3244	case 2: /* sixteen banks */
3245	default:
3246		rdev->config.si.tile_config |= 2 << 4;
3247		break;
3248	}
3249	rdev->config.si.tile_config |=
3250		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3251	rdev->config.si.tile_config |=
3252		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3253
3254	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3255	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3256	WREG32(DMIF_ADDR_CALC, gb_addr_config);
3257	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3258	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3259	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3260	if (rdev->has_uvd) {
3261		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3262		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3263		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3264	}
3265
3266	si_tiling_mode_table_init(rdev);
3267
3268	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3269		    rdev->config.si.max_sh_per_se,
3270		    rdev->config.si.max_backends_per_se);
3271
3272	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3273		     rdev->config.si.max_sh_per_se,
3274		     rdev->config.si.max_cu_per_sh);
3275
3276	rdev->config.si.active_cus = 0;
3277	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3278		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3279			rdev->config.si.active_cus +=
3280				hweight32(si_get_cu_active_bitmap(rdev, i, j));
3281		}
3282	}
3283
3284	/* set HW defaults for 3D engine */
3285	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3286				     ROQ_IB2_START(0x2b)));
3287	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3288
3289	sx_debug_1 = RREG32(SX_DEBUG_1);
3290	WREG32(SX_DEBUG_1, sx_debug_1);
3291
3292	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3293
3294	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3295				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3296				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3297				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3298
3299	WREG32(VGT_NUM_INSTANCES, 1);
3300
3301	WREG32(CP_PERFMON_CNTL, 0);
3302
3303	WREG32(SQ_CONFIG, 0);
3304
3305	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3306					  FORCE_EOV_MAX_REZ_CNT(255)));
3307
3308	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3309	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
3310
3311	WREG32(VGT_GS_VERTEX_REUSE, 16);
3312	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3313
3314	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3315	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3316	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3317	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3318	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3319	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3320	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3321	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3322
3323	tmp = RREG32(HDP_MISC_CNTL);
3324	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3325	WREG32(HDP_MISC_CNTL, tmp);
3326
3327	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3328	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3329
3330	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3331
3332	udelay(50);
3333}
3334
3335/*
3336 * GPU scratch registers helpers function.
3337 */
3338static void si_scratch_init(struct radeon_device *rdev)
3339{
3340	int i;
3341
3342	rdev->scratch.num_reg = 7;
3343	rdev->scratch.reg_base = SCRATCH_REG0;
3344	for (i = 0; i < rdev->scratch.num_reg; i++) {
3345		rdev->scratch.free[i] = true;
3346		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3347	}
3348}
3349
3350void si_fence_ring_emit(struct radeon_device *rdev,
3351			struct radeon_fence *fence)
3352{
3353	struct radeon_ring *ring = &rdev->ring[fence->ring];
3354	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3355
3356	/* flush read cache over gart */
3357	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3358	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3359	radeon_ring_write(ring, 0);
3360	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3361	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3362			  PACKET3_TC_ACTION_ENA |
3363			  PACKET3_SH_KCACHE_ACTION_ENA |
3364			  PACKET3_SH_ICACHE_ACTION_ENA);
3365	radeon_ring_write(ring, 0xFFFFFFFF);
3366	radeon_ring_write(ring, 0);
3367	radeon_ring_write(ring, 10); /* poll interval */
3368	/* EVENT_WRITE_EOP - flush caches, send int */
3369	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3370	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3371	radeon_ring_write(ring, lower_32_bits(addr));
3372	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3373	radeon_ring_write(ring, fence->seq);
3374	radeon_ring_write(ring, 0);
3375}
3376
3377/*
3378 * IB stuff
3379 */
3380void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3381{
3382	struct radeon_ring *ring = &rdev->ring[ib->ring];
3383	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3384	u32 header;
3385
3386	if (ib->is_const_ib) {
3387		/* set switch buffer packet before const IB */
3388		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3389		radeon_ring_write(ring, 0);
3390
3391		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3392	} else {
3393		u32 next_rptr;
3394		if (ring->rptr_save_reg) {
3395			next_rptr = ring->wptr + 3 + 4 + 8;
3396			radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3397			radeon_ring_write(ring, ((ring->rptr_save_reg -
3398						  PACKET3_SET_CONFIG_REG_START) >> 2));
3399			radeon_ring_write(ring, next_rptr);
3400		} else if (rdev->wb.enabled) {
3401			next_rptr = ring->wptr + 5 + 4 + 8;
3402			radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3403			radeon_ring_write(ring, (1 << 8));
3404			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3405			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3406			radeon_ring_write(ring, next_rptr);
3407		}
3408
3409		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3410	}
3411
3412	radeon_ring_write(ring, header);
3413	radeon_ring_write(ring,
3414#ifdef __BIG_ENDIAN
3415			  (2 << 0) |
3416#endif
3417			  (ib->gpu_addr & 0xFFFFFFFC));
3418	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3419	radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3420
3421	if (!ib->is_const_ib) {
3422		/* flush read cache over gart for this vmid */
3423		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3424		radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3425		radeon_ring_write(ring, vm_id);
3426		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3427		radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3428				  PACKET3_TC_ACTION_ENA |
3429				  PACKET3_SH_KCACHE_ACTION_ENA |
3430				  PACKET3_SH_ICACHE_ACTION_ENA);
3431		radeon_ring_write(ring, 0xFFFFFFFF);
3432		radeon_ring_write(ring, 0);
3433		radeon_ring_write(ring, 10); /* poll interval */
3434	}
3435}
3436
3437/*
3438 * CP.
3439 */
3440static void si_cp_enable(struct radeon_device *rdev, bool enable)
3441{
3442	if (enable)
3443		WREG32(CP_ME_CNTL, 0);
3444	else {
3445		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3446			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3447		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3448		WREG32(SCRATCH_UMSK, 0);
3449		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3450		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3451		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3452	}
3453	udelay(50);
3454}
3455
3456static int si_cp_load_microcode(struct radeon_device *rdev)
3457{
3458	int i;
3459
3460	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3461		return -EINVAL;
3462
3463	si_cp_enable(rdev, false);
3464
3465	if (rdev->new_fw) {
3466		const struct gfx_firmware_header_v1_0 *pfp_hdr =
3467			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3468		const struct gfx_firmware_header_v1_0 *ce_hdr =
3469			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3470		const struct gfx_firmware_header_v1_0 *me_hdr =
3471			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3472		const __le32 *fw_data;
3473		u32 fw_size;
3474
3475		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3476		radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3477		radeon_ucode_print_gfx_hdr(&me_hdr->header);
3478
3479		/* PFP */
3480		fw_data = (const __le32 *)
3481			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3482		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3483		WREG32(CP_PFP_UCODE_ADDR, 0);
3484		for (i = 0; i < fw_size; i++)
3485			WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3486		WREG32(CP_PFP_UCODE_ADDR, 0);
3487
3488		/* CE */
3489		fw_data = (const __le32 *)
3490			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3491		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3492		WREG32(CP_CE_UCODE_ADDR, 0);
3493		for (i = 0; i < fw_size; i++)
3494			WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3495		WREG32(CP_CE_UCODE_ADDR, 0);
3496
3497		/* ME */
3498		fw_data = (const __be32 *)
3499			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3500		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3501		WREG32(CP_ME_RAM_WADDR, 0);
3502		for (i = 0; i < fw_size; i++)
3503			WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3504		WREG32(CP_ME_RAM_WADDR, 0);
3505	} else {
3506		const __be32 *fw_data;
3507
3508		/* PFP */
3509		fw_data = (const __be32 *)rdev->pfp_fw->data;
3510		WREG32(CP_PFP_UCODE_ADDR, 0);
3511		for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3512			WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3513		WREG32(CP_PFP_UCODE_ADDR, 0);
3514
3515		/* CE */
3516		fw_data = (const __be32 *)rdev->ce_fw->data;
3517		WREG32(CP_CE_UCODE_ADDR, 0);
3518		for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3519			WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3520		WREG32(CP_CE_UCODE_ADDR, 0);
3521
3522		/* ME */
3523		fw_data = (const __be32 *)rdev->me_fw->data;
3524		WREG32(CP_ME_RAM_WADDR, 0);
3525		for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3526			WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3527		WREG32(CP_ME_RAM_WADDR, 0);
3528	}
3529
3530	WREG32(CP_PFP_UCODE_ADDR, 0);
3531	WREG32(CP_CE_UCODE_ADDR, 0);
3532	WREG32(CP_ME_RAM_WADDR, 0);
3533	WREG32(CP_ME_RAM_RADDR, 0);
3534	return 0;
3535}
3536
3537static int si_cp_start(struct radeon_device *rdev)
3538{
3539	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3540	int r, i;
3541
3542	r = radeon_ring_lock(rdev, ring, 7 + 4);
3543	if (r) {
3544		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3545		return r;
3546	}
3547	/* init the CP */
3548	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3549	radeon_ring_write(ring, 0x1);
3550	radeon_ring_write(ring, 0x0);
3551	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3552	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3553	radeon_ring_write(ring, 0);
3554	radeon_ring_write(ring, 0);
3555
3556	/* init the CE partitions */
3557	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3558	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3559	radeon_ring_write(ring, 0xc000);
3560	radeon_ring_write(ring, 0xe000);
3561	radeon_ring_unlock_commit(rdev, ring, false);
3562
3563	si_cp_enable(rdev, true);
3564
3565	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3566	if (r) {
3567		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3568		return r;
3569	}
3570
3571	/* setup clear context state */
3572	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3573	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3574
3575	for (i = 0; i < si_default_size; i++)
3576		radeon_ring_write(ring, si_default_state[i]);
3577
3578	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3579	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3580
3581	/* set clear context state */
3582	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3583	radeon_ring_write(ring, 0);
3584
3585	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3586	radeon_ring_write(ring, 0x00000316);
3587	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3588	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3589
3590	radeon_ring_unlock_commit(rdev, ring, false);
3591
3592	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3593		ring = &rdev->ring[i];
3594		r = radeon_ring_lock(rdev, ring, 2);
3595		if (r) {
3596			DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3597			return r;
3598		}
3599
3600		/* clear the compute context state */
3601		radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3602		radeon_ring_write(ring, 0);
3603
3604		radeon_ring_unlock_commit(rdev, ring, false);
3605	}
3606
3607	return 0;
3608}
3609
3610static void si_cp_fini(struct radeon_device *rdev)
3611{
3612	struct radeon_ring *ring;
3613	si_cp_enable(rdev, false);
3614
3615	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3616	radeon_ring_fini(rdev, ring);
3617	radeon_scratch_free(rdev, ring->rptr_save_reg);
3618
3619	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3620	radeon_ring_fini(rdev, ring);
3621	radeon_scratch_free(rdev, ring->rptr_save_reg);
3622
3623	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3624	radeon_ring_fini(rdev, ring);
3625	radeon_scratch_free(rdev, ring->rptr_save_reg);
3626}
3627
3628static int si_cp_resume(struct radeon_device *rdev)
3629{
3630	struct radeon_ring *ring;
3631	u32 tmp;
3632	u32 rb_bufsz;
3633	int r;
3634
3635	si_enable_gui_idle_interrupt(rdev, false);
3636
3637	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3638	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3639
3640	/* Set the write pointer delay */
3641	WREG32(CP_RB_WPTR_DELAY, 0);
3642
3643	WREG32(CP_DEBUG, 0);
3644	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3645
3646	/* ring 0 - compute and gfx */
3647	/* Set ring buffer size */
3648	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3649	rb_bufsz = order_base_2(ring->ring_size / 8);
3650	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3651#ifdef __BIG_ENDIAN
3652	tmp |= BUF_SWAP_32BIT;
3653#endif
3654	WREG32(CP_RB0_CNTL, tmp);
3655
3656	/* Initialize the ring buffer's read and write pointers */
3657	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3658	ring->wptr = 0;
3659	WREG32(CP_RB0_WPTR, ring->wptr);
3660
3661	/* set the wb address whether it's enabled or not */
3662	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3663	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3664
3665	if (rdev->wb.enabled)
3666		WREG32(SCRATCH_UMSK, 0xff);
3667	else {
3668		tmp |= RB_NO_UPDATE;
3669		WREG32(SCRATCH_UMSK, 0);
3670	}
3671
3672	mdelay(1);
3673	WREG32(CP_RB0_CNTL, tmp);
3674
3675	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3676
3677	/* ring1  - compute only */
3678	/* Set ring buffer size */
3679	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3680	rb_bufsz = order_base_2(ring->ring_size / 8);
3681	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3682#ifdef __BIG_ENDIAN
3683	tmp |= BUF_SWAP_32BIT;
3684#endif
3685	WREG32(CP_RB1_CNTL, tmp);
3686
3687	/* Initialize the ring buffer's read and write pointers */
3688	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3689	ring->wptr = 0;
3690	WREG32(CP_RB1_WPTR, ring->wptr);
3691
3692	/* set the wb address whether it's enabled or not */
3693	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3694	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3695
3696	mdelay(1);
3697	WREG32(CP_RB1_CNTL, tmp);
3698
3699	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3700
3701	/* ring2 - compute only */
3702	/* Set ring buffer size */
3703	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3704	rb_bufsz = order_base_2(ring->ring_size / 8);
3705	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3706#ifdef __BIG_ENDIAN
3707	tmp |= BUF_SWAP_32BIT;
3708#endif
3709	WREG32(CP_RB2_CNTL, tmp);
3710
3711	/* Initialize the ring buffer's read and write pointers */
3712	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3713	ring->wptr = 0;
3714	WREG32(CP_RB2_WPTR, ring->wptr);
3715
3716	/* set the wb address whether it's enabled or not */
3717	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3718	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3719
3720	mdelay(1);
3721	WREG32(CP_RB2_CNTL, tmp);
3722
3723	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3724
3725	/* start the rings */
3726	si_cp_start(rdev);
3727	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3728	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3729	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3730	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3731	if (r) {
3732		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3733		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3734		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3735		return r;
3736	}
3737	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3738	if (r) {
3739		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3740	}
3741	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3742	if (r) {
3743		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3744	}
3745
3746	si_enable_gui_idle_interrupt(rdev, true);
3747
3748	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3749		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3750
3751	return 0;
3752}
3753
3754u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3755{
3756	u32 reset_mask = 0;
3757	u32 tmp;
3758
3759	/* GRBM_STATUS */
3760	tmp = RREG32(GRBM_STATUS);
3761	if (tmp & (PA_BUSY | SC_BUSY |
3762		   BCI_BUSY | SX_BUSY |
3763		   TA_BUSY | VGT_BUSY |
3764		   DB_BUSY | CB_BUSY |
3765		   GDS_BUSY | SPI_BUSY |
3766		   IA_BUSY | IA_BUSY_NO_DMA))
3767		reset_mask |= RADEON_RESET_GFX;
3768
3769	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3770		   CP_BUSY | CP_COHERENCY_BUSY))
3771		reset_mask |= RADEON_RESET_CP;
3772
3773	if (tmp & GRBM_EE_BUSY)
3774		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3775
3776	/* GRBM_STATUS2 */
3777	tmp = RREG32(GRBM_STATUS2);
3778	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3779		reset_mask |= RADEON_RESET_RLC;
3780
3781	/* DMA_STATUS_REG 0 */
3782	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3783	if (!(tmp & DMA_IDLE))
3784		reset_mask |= RADEON_RESET_DMA;
3785
3786	/* DMA_STATUS_REG 1 */
3787	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3788	if (!(tmp & DMA_IDLE))
3789		reset_mask |= RADEON_RESET_DMA1;
3790
3791	/* SRBM_STATUS2 */
3792	tmp = RREG32(SRBM_STATUS2);
3793	if (tmp & DMA_BUSY)
3794		reset_mask |= RADEON_RESET_DMA;
3795
3796	if (tmp & DMA1_BUSY)
3797		reset_mask |= RADEON_RESET_DMA1;
3798
3799	/* SRBM_STATUS */
3800	tmp = RREG32(SRBM_STATUS);
3801
3802	if (tmp & IH_BUSY)
3803		reset_mask |= RADEON_RESET_IH;
3804
3805	if (tmp & SEM_BUSY)
3806		reset_mask |= RADEON_RESET_SEM;
3807
3808	if (tmp & GRBM_RQ_PENDING)
3809		reset_mask |= RADEON_RESET_GRBM;
3810
3811	if (tmp & VMC_BUSY)
3812		reset_mask |= RADEON_RESET_VMC;
3813
3814	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3815		   MCC_BUSY | MCD_BUSY))
3816		reset_mask |= RADEON_RESET_MC;
3817
3818	if (evergreen_is_display_hung(rdev))
3819		reset_mask |= RADEON_RESET_DISPLAY;
3820
3821	/* VM_L2_STATUS */
3822	tmp = RREG32(VM_L2_STATUS);
3823	if (tmp & L2_BUSY)
3824		reset_mask |= RADEON_RESET_VMC;
3825
3826	/* Skip MC reset as it's mostly likely not hung, just busy */
3827	if (reset_mask & RADEON_RESET_MC) {
3828		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3829		reset_mask &= ~RADEON_RESET_MC;
3830	}
3831
3832	return reset_mask;
3833}
3834
3835static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3836{
3837	struct evergreen_mc_save save;
3838	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3839	u32 tmp;
3840
3841	if (reset_mask == 0)
3842		return;
3843
3844	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3845
3846	evergreen_print_gpu_status_regs(rdev);
3847	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3848		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3849	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3850		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3851
3852	/* disable PG/CG */
3853	si_fini_pg(rdev);
3854	si_fini_cg(rdev);
3855
3856	/* stop the rlc */
3857	si_rlc_stop(rdev);
3858
3859	/* Disable CP parsing/prefetching */
3860	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3861
3862	if (reset_mask & RADEON_RESET_DMA) {
3863		/* dma0 */
3864		tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3865		tmp &= ~DMA_RB_ENABLE;
3866		WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3867	}
3868	if (reset_mask & RADEON_RESET_DMA1) {
3869		/* dma1 */
3870		tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3871		tmp &= ~DMA_RB_ENABLE;
3872		WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3873	}
3874
3875	udelay(50);
3876
3877	evergreen_mc_stop(rdev, &save);
3878	if (evergreen_mc_wait_for_idle(rdev)) {
3879		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3880	}
3881
3882	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3883		grbm_soft_reset = SOFT_RESET_CB |
3884			SOFT_RESET_DB |
3885			SOFT_RESET_GDS |
3886			SOFT_RESET_PA |
3887			SOFT_RESET_SC |
3888			SOFT_RESET_BCI |
3889			SOFT_RESET_SPI |
3890			SOFT_RESET_SX |
3891			SOFT_RESET_TC |
3892			SOFT_RESET_TA |
3893			SOFT_RESET_VGT |
3894			SOFT_RESET_IA;
3895	}
3896
3897	if (reset_mask & RADEON_RESET_CP) {
3898		grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3899
3900		srbm_soft_reset |= SOFT_RESET_GRBM;
3901	}
3902
3903	if (reset_mask & RADEON_RESET_DMA)
3904		srbm_soft_reset |= SOFT_RESET_DMA;
3905
3906	if (reset_mask & RADEON_RESET_DMA1)
3907		srbm_soft_reset |= SOFT_RESET_DMA1;
3908
3909	if (reset_mask & RADEON_RESET_DISPLAY)
3910		srbm_soft_reset |= SOFT_RESET_DC;
3911
3912	if (reset_mask & RADEON_RESET_RLC)
3913		grbm_soft_reset |= SOFT_RESET_RLC;
3914
3915	if (reset_mask & RADEON_RESET_SEM)
3916		srbm_soft_reset |= SOFT_RESET_SEM;
3917
3918	if (reset_mask & RADEON_RESET_IH)
3919		srbm_soft_reset |= SOFT_RESET_IH;
3920
3921	if (reset_mask & RADEON_RESET_GRBM)
3922		srbm_soft_reset |= SOFT_RESET_GRBM;
3923
3924	if (reset_mask & RADEON_RESET_VMC)
3925		srbm_soft_reset |= SOFT_RESET_VMC;
3926
3927	if (reset_mask & RADEON_RESET_MC)
3928		srbm_soft_reset |= SOFT_RESET_MC;
3929
3930	if (grbm_soft_reset) {
3931		tmp = RREG32(GRBM_SOFT_RESET);
3932		tmp |= grbm_soft_reset;
3933		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3934		WREG32(GRBM_SOFT_RESET, tmp);
3935		tmp = RREG32(GRBM_SOFT_RESET);
3936
3937		udelay(50);
3938
3939		tmp &= ~grbm_soft_reset;
3940		WREG32(GRBM_SOFT_RESET, tmp);
3941		tmp = RREG32(GRBM_SOFT_RESET);
3942	}
3943
3944	if (srbm_soft_reset) {
3945		tmp = RREG32(SRBM_SOFT_RESET);
3946		tmp |= srbm_soft_reset;
3947		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3948		WREG32(SRBM_SOFT_RESET, tmp);
3949		tmp = RREG32(SRBM_SOFT_RESET);
3950
3951		udelay(50);
3952
3953		tmp &= ~srbm_soft_reset;
3954		WREG32(SRBM_SOFT_RESET, tmp);
3955		tmp = RREG32(SRBM_SOFT_RESET);
3956	}
3957
3958	/* Wait a little for things to settle down */
3959	udelay(50);
3960
3961	evergreen_mc_resume(rdev, &save);
3962	udelay(50);
3963
3964	evergreen_print_gpu_status_regs(rdev);
3965}
3966
3967static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3968{
3969	u32 tmp, i;
3970
3971	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3972	tmp |= SPLL_BYPASS_EN;
3973	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3974
3975	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3976	tmp |= SPLL_CTLREQ_CHG;
3977	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3978
3979	for (i = 0; i < rdev->usec_timeout; i++) {
3980		if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3981			break;
3982		udelay(1);
3983	}
3984
3985	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3986	tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3987	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3988
3989	tmp = RREG32(MPLL_CNTL_MODE);
3990	tmp &= ~MPLL_MCLK_SEL;
3991	WREG32(MPLL_CNTL_MODE, tmp);
3992}
3993
3994static void si_spll_powerdown(struct radeon_device *rdev)
3995{
3996	u32 tmp;
3997
3998	tmp = RREG32(SPLL_CNTL_MODE);
3999	tmp |= SPLL_SW_DIR_CONTROL;
4000	WREG32(SPLL_CNTL_MODE, tmp);
4001
4002	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4003	tmp |= SPLL_RESET;
4004	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4005
4006	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4007	tmp |= SPLL_SLEEP;
4008	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4009
4010	tmp = RREG32(SPLL_CNTL_MODE);
4011	tmp &= ~SPLL_SW_DIR_CONTROL;
4012	WREG32(SPLL_CNTL_MODE, tmp);
4013}
4014
4015static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4016{
4017	struct evergreen_mc_save save;
4018	u32 tmp, i;
4019
4020	dev_info(rdev->dev, "GPU pci config reset\n");
4021
4022	/* disable dpm? */
4023
4024	/* disable cg/pg */
4025	si_fini_pg(rdev);
4026	si_fini_cg(rdev);
4027
4028	/* Disable CP parsing/prefetching */
4029	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4030	/* dma0 */
4031	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4032	tmp &= ~DMA_RB_ENABLE;
4033	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4034	/* dma1 */
4035	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4036	tmp &= ~DMA_RB_ENABLE;
4037	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4038	/* XXX other engines? */
4039
4040	/* halt the rlc, disable cp internal ints */
4041	si_rlc_stop(rdev);
4042
4043	udelay(50);
4044
4045	/* disable mem access */
4046	evergreen_mc_stop(rdev, &save);
4047	if (evergreen_mc_wait_for_idle(rdev)) {
4048		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4049	}
4050
4051	/* set mclk/sclk to bypass */
4052	si_set_clk_bypass_mode(rdev);
4053	/* powerdown spll */
4054	si_spll_powerdown(rdev);
4055	/* disable BM */
4056	pci_clear_master(rdev->pdev);
4057	/* reset */
4058	radeon_pci_config_reset(rdev);
4059	/* wait for asic to come out of reset */
4060	for (i = 0; i < rdev->usec_timeout; i++) {
4061		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4062			break;
4063		udelay(1);
4064	}
4065}
4066
4067int si_asic_reset(struct radeon_device *rdev, bool hard)
4068{
4069	u32 reset_mask;
4070
4071	if (hard) {
4072		si_gpu_pci_config_reset(rdev);
4073		return 0;
4074	}
4075
4076	reset_mask = si_gpu_check_soft_reset(rdev);
4077
4078	if (reset_mask)
4079		r600_set_bios_scratch_engine_hung(rdev, true);
4080
4081	/* try soft reset */
4082	si_gpu_soft_reset(rdev, reset_mask);
4083
4084	reset_mask = si_gpu_check_soft_reset(rdev);
4085
4086	/* try pci config reset */
4087	if (reset_mask && radeon_hard_reset)
4088		si_gpu_pci_config_reset(rdev);
4089
4090	reset_mask = si_gpu_check_soft_reset(rdev);
4091
4092	if (!reset_mask)
4093		r600_set_bios_scratch_engine_hung(rdev, false);
4094
4095	return 0;
4096}
4097
4098/**
4099 * si_gfx_is_lockup - Check if the GFX engine is locked up
4100 *
4101 * @rdev: radeon_device pointer
4102 * @ring: radeon_ring structure holding ring information
4103 *
4104 * Check if the GFX engine is locked up.
4105 * Returns true if the engine appears to be locked up, false if not.
4106 */
4107bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4108{
4109	u32 reset_mask = si_gpu_check_soft_reset(rdev);
4110
4111	if (!(reset_mask & (RADEON_RESET_GFX |
4112			    RADEON_RESET_COMPUTE |
4113			    RADEON_RESET_CP))) {
4114		radeon_ring_lockup_update(rdev, ring);
4115		return false;
4116	}
4117	return radeon_ring_test_lockup(rdev, ring);
4118}
4119
4120/* MC */
4121static void si_mc_program(struct radeon_device *rdev)
4122{
4123	struct evergreen_mc_save save;
4124	u32 tmp;
4125	int i, j;
4126
4127	/* Initialize HDP */
4128	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4129		WREG32((0x2c14 + j), 0x00000000);
4130		WREG32((0x2c18 + j), 0x00000000);
4131		WREG32((0x2c1c + j), 0x00000000);
4132		WREG32((0x2c20 + j), 0x00000000);
4133		WREG32((0x2c24 + j), 0x00000000);
4134	}
4135	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4136
4137	evergreen_mc_stop(rdev, &save);
4138	if (radeon_mc_wait_for_idle(rdev)) {
4139		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4140	}
4141	if (!ASIC_IS_NODCE(rdev))
4142		/* Lockout access through VGA aperture*/
4143		WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4144	/* Update configuration */
4145	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4146	       rdev->mc.vram_start >> 12);
4147	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4148	       rdev->mc.vram_end >> 12);
4149	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4150	       rdev->vram_scratch.gpu_addr >> 12);
4151	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4152	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4153	WREG32(MC_VM_FB_LOCATION, tmp);
4154	/* XXX double check these! */
4155	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4156	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4157	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4158	WREG32(MC_VM_AGP_BASE, 0);
4159	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4160	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4161	if (radeon_mc_wait_for_idle(rdev)) {
4162		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4163	}
4164	evergreen_mc_resume(rdev, &save);
4165	if (!ASIC_IS_NODCE(rdev)) {
4166		/* we need to own VRAM, so turn off the VGA renderer here
4167		 * to stop it overwriting our objects */
4168		rv515_vga_render_disable(rdev);
4169	}
4170}
4171
4172void si_vram_gtt_location(struct radeon_device *rdev,
4173			  struct radeon_mc *mc)
4174{
4175	if (mc->mc_vram_size > 0xFFC0000000ULL) {
4176		/* leave room for at least 1024M GTT */
4177		dev_warn(rdev->dev, "limiting VRAM\n");
4178		mc->real_vram_size = 0xFFC0000000ULL;
4179		mc->mc_vram_size = 0xFFC0000000ULL;
4180	}
4181	radeon_vram_location(rdev, &rdev->mc, 0);
4182	rdev->mc.gtt_base_align = 0;
4183	radeon_gtt_location(rdev, mc);
4184}
4185
4186static int si_mc_init(struct radeon_device *rdev)
4187{
4188	u32 tmp;
4189	int chansize, numchan;
4190
4191	/* Get VRAM informations */
4192	rdev->mc.vram_is_ddr = true;
4193	tmp = RREG32(MC_ARB_RAMCFG);
4194	if (tmp & CHANSIZE_OVERRIDE) {
4195		chansize = 16;
4196	} else if (tmp & CHANSIZE_MASK) {
4197		chansize = 64;
4198	} else {
4199		chansize = 32;
4200	}
4201	tmp = RREG32(MC_SHARED_CHMAP);
4202	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4203	case 0:
4204	default:
4205		numchan = 1;
4206		break;
4207	case 1:
4208		numchan = 2;
4209		break;
4210	case 2:
4211		numchan = 4;
4212		break;
4213	case 3:
4214		numchan = 8;
4215		break;
4216	case 4:
4217		numchan = 3;
4218		break;
4219	case 5:
4220		numchan = 6;
4221		break;
4222	case 6:
4223		numchan = 10;
4224		break;
4225	case 7:
4226		numchan = 12;
4227		break;
4228	case 8:
4229		numchan = 16;
4230		break;
4231	}
4232	rdev->mc.vram_width = numchan * chansize;
4233	/* Could aper size report 0 ? */
4234	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4235	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4236	/* size in MB on si */
4237	tmp = RREG32(CONFIG_MEMSIZE);
4238	/* some boards may have garbage in the upper 16 bits */
4239	if (tmp & 0xffff0000) {
4240		DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4241		if (tmp & 0xffff)
4242			tmp &= 0xffff;
4243	}
4244	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4245	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4246	rdev->mc.visible_vram_size = rdev->mc.aper_size;
4247	si_vram_gtt_location(rdev, &rdev->mc);
4248	radeon_update_bandwidth_info(rdev);
4249
4250	return 0;
4251}
4252
4253/*
4254 * GART
4255 */
4256void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4257{
4258	/* flush hdp cache */
4259	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4260
4261	/* bits 0-15 are the VM contexts0-15 */
4262	WREG32(VM_INVALIDATE_REQUEST, 1);
4263}
4264
4265static int si_pcie_gart_enable(struct radeon_device *rdev)
4266{
4267	int r, i;
4268
4269	if (rdev->gart.robj == NULL) {
4270		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4271		return -EINVAL;
4272	}
4273	r = radeon_gart_table_vram_pin(rdev);
4274	if (r)
4275		return r;
4276	/* Setup TLB control */
4277	WREG32(MC_VM_MX_L1_TLB_CNTL,
4278	       (0xA << 7) |
4279	       ENABLE_L1_TLB |
4280	       ENABLE_L1_FRAGMENT_PROCESSING |
4281	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4282	       ENABLE_ADVANCED_DRIVER_MODEL |
4283	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4284	/* Setup L2 cache */
4285	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4286	       ENABLE_L2_FRAGMENT_PROCESSING |
4287	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4288	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4289	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4290	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4291	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4292	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4293	       BANK_SELECT(4) |
4294	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4295	/* setup context0 */
4296	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4297	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4298	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4299	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4300			(u32)(rdev->dummy_page.addr >> 12));
4301	WREG32(VM_CONTEXT0_CNTL2, 0);
4302	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4303				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4304
4305	WREG32(0x15D4, 0);
4306	WREG32(0x15D8, 0);
4307	WREG32(0x15DC, 0);
4308
4309	/* empty context1-15 */
4310	/* set vm size, must be a multiple of 4 */
4311	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4312	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4313	/* Assign the pt base to something valid for now; the pts used for
4314	 * the VMs are determined by the application and setup and assigned
4315	 * on the fly in the vm part of radeon_gart.c
4316	 */
4317	for (i = 1; i < 16; i++) {
4318		if (i < 8)
4319			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4320			       rdev->vm_manager.saved_table_addr[i]);
4321		else
4322			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4323			       rdev->vm_manager.saved_table_addr[i]);
4324	}
4325
4326	/* enable context1-15 */
4327	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4328	       (u32)(rdev->dummy_page.addr >> 12));
4329	WREG32(VM_CONTEXT1_CNTL2, 4);
4330	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4331				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4332				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4333				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4334				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4335				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4336				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4337				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4338				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4339				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4340				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4341				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4342				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4343				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4344
4345	si_pcie_gart_tlb_flush(rdev);
4346	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4347		 (unsigned)(rdev->mc.gtt_size >> 20),
4348		 (unsigned long long)rdev->gart.table_addr);
4349	rdev->gart.ready = true;
4350	return 0;
4351}
4352
4353static void si_pcie_gart_disable(struct radeon_device *rdev)
4354{
4355	unsigned i;
4356
4357	for (i = 1; i < 16; ++i) {
4358		uint32_t reg;
4359		if (i < 8)
4360			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4361		else
4362			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4363		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4364	}
4365
4366	/* Disable all tables */
4367	WREG32(VM_CONTEXT0_CNTL, 0);
4368	WREG32(VM_CONTEXT1_CNTL, 0);
4369	/* Setup TLB control */
4370	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4371	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4372	/* Setup L2 cache */
4373	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4374	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4375	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4376	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4377	WREG32(VM_L2_CNTL2, 0);
4378	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4379	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4380	radeon_gart_table_vram_unpin(rdev);
4381}
4382
4383static void si_pcie_gart_fini(struct radeon_device *rdev)
4384{
4385	si_pcie_gart_disable(rdev);
4386	radeon_gart_table_vram_free(rdev);
4387	radeon_gart_fini(rdev);
4388}
4389
4390/* vm parser */
4391static bool si_vm_reg_valid(u32 reg)
4392{
4393	/* context regs are fine */
4394	if (reg >= 0x28000)
4395		return true;
4396
4397	/* shader regs are also fine */
4398	if (reg >= 0xB000 && reg < 0xC000)
4399		return true;
4400
4401	/* check config regs */
4402	switch (reg) {
4403	case GRBM_GFX_INDEX:
4404	case CP_STRMOUT_CNTL:
4405	case VGT_VTX_VECT_EJECT_REG:
4406	case VGT_CACHE_INVALIDATION:
4407	case VGT_ESGS_RING_SIZE:
4408	case VGT_GSVS_RING_SIZE:
4409	case VGT_GS_VERTEX_REUSE:
4410	case VGT_PRIMITIVE_TYPE:
4411	case VGT_INDEX_TYPE:
4412	case VGT_NUM_INDICES:
4413	case VGT_NUM_INSTANCES:
4414	case VGT_TF_RING_SIZE:
4415	case VGT_HS_OFFCHIP_PARAM:
4416	case VGT_TF_MEMORY_BASE:
4417	case PA_CL_ENHANCE:
4418	case PA_SU_LINE_STIPPLE_VALUE:
4419	case PA_SC_LINE_STIPPLE_STATE:
4420	case PA_SC_ENHANCE:
4421	case SQC_CACHES:
4422	case SPI_STATIC_THREAD_MGMT_1:
4423	case SPI_STATIC_THREAD_MGMT_2:
4424	case SPI_STATIC_THREAD_MGMT_3:
4425	case SPI_PS_MAX_WAVE_ID:
4426	case SPI_CONFIG_CNTL:
4427	case SPI_CONFIG_CNTL_1:
4428	case TA_CNTL_AUX:
4429	case TA_CS_BC_BASE_ADDR:
4430		return true;
4431	default:
4432		DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4433		return false;
4434	}
4435}
4436
4437static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4438				  u32 *ib, struct radeon_cs_packet *pkt)
4439{
4440	switch (pkt->opcode) {
4441	case PACKET3_NOP:
4442	case PACKET3_SET_BASE:
4443	case PACKET3_SET_CE_DE_COUNTERS:
4444	case PACKET3_LOAD_CONST_RAM:
4445	case PACKET3_WRITE_CONST_RAM:
4446	case PACKET3_WRITE_CONST_RAM_OFFSET:
4447	case PACKET3_DUMP_CONST_RAM:
4448	case PACKET3_INCREMENT_CE_COUNTER:
4449	case PACKET3_WAIT_ON_DE_COUNTER:
4450	case PACKET3_CE_WRITE:
4451		break;
4452	default:
4453		DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4454		return -EINVAL;
4455	}
4456	return 0;
4457}
4458
4459static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4460{
4461	u32 start_reg, reg, i;
4462	u32 command = ib[idx + 4];
4463	u32 info = ib[idx + 1];
4464	u32 idx_value = ib[idx];
4465	if (command & PACKET3_CP_DMA_CMD_SAS) {
4466		/* src address space is register */
4467		if (((info & 0x60000000) >> 29) == 0) {
4468			start_reg = idx_value << 2;
4469			if (command & PACKET3_CP_DMA_CMD_SAIC) {
4470				reg = start_reg;
4471				if (!si_vm_reg_valid(reg)) {
4472					DRM_ERROR("CP DMA Bad SRC register\n");
4473					return -EINVAL;
4474				}
4475			} else {
4476				for (i = 0; i < (command & 0x1fffff); i++) {
4477					reg = start_reg + (4 * i);
4478					if (!si_vm_reg_valid(reg)) {
4479						DRM_ERROR("CP DMA Bad SRC register\n");
4480						return -EINVAL;
4481					}
4482				}
4483			}
4484		}
4485	}
4486	if (command & PACKET3_CP_DMA_CMD_DAS) {
4487		/* dst address space is register */
4488		if (((info & 0x00300000) >> 20) == 0) {
4489			start_reg = ib[idx + 2];
4490			if (command & PACKET3_CP_DMA_CMD_DAIC) {
4491				reg = start_reg;
4492				if (!si_vm_reg_valid(reg)) {
4493					DRM_ERROR("CP DMA Bad DST register\n");
4494					return -EINVAL;
4495				}
4496			} else {
4497				for (i = 0; i < (command & 0x1fffff); i++) {
4498					reg = start_reg + (4 * i);
4499					if (!si_vm_reg_valid(reg)) {
4500						DRM_ERROR("CP DMA Bad DST register\n");
4501						return -EINVAL;
4502					}
4503				}
4504			}
4505		}
4506	}
4507	return 0;
4508}
4509
4510static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4511				   u32 *ib, struct radeon_cs_packet *pkt)
4512{
4513	int r;
4514	u32 idx = pkt->idx + 1;
4515	u32 idx_value = ib[idx];
4516	u32 start_reg, end_reg, reg, i;
4517
4518	switch (pkt->opcode) {
4519	case PACKET3_NOP:
4520	case PACKET3_SET_BASE:
4521	case PACKET3_CLEAR_STATE:
4522	case PACKET3_INDEX_BUFFER_SIZE:
4523	case PACKET3_DISPATCH_DIRECT:
4524	case PACKET3_DISPATCH_INDIRECT:
4525	case PACKET3_ALLOC_GDS:
4526	case PACKET3_WRITE_GDS_RAM:
4527	case PACKET3_ATOMIC_GDS:
4528	case PACKET3_ATOMIC:
4529	case PACKET3_OCCLUSION_QUERY:
4530	case PACKET3_SET_PREDICATION:
4531	case PACKET3_COND_EXEC:
4532	case PACKET3_PRED_EXEC:
4533	case PACKET3_DRAW_INDIRECT:
4534	case PACKET3_DRAW_INDEX_INDIRECT:
4535	case PACKET3_INDEX_BASE:
4536	case PACKET3_DRAW_INDEX_2:
4537	case PACKET3_CONTEXT_CONTROL:
4538	case PACKET3_INDEX_TYPE:
4539	case PACKET3_DRAW_INDIRECT_MULTI:
4540	case PACKET3_DRAW_INDEX_AUTO:
4541	case PACKET3_DRAW_INDEX_IMMD:
4542	case PACKET3_NUM_INSTANCES:
4543	case PACKET3_DRAW_INDEX_MULTI_AUTO:
4544	case PACKET3_STRMOUT_BUFFER_UPDATE:
4545	case PACKET3_DRAW_INDEX_OFFSET_2:
4546	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4547	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4548	case PACKET3_MPEG_INDEX:
4549	case PACKET3_WAIT_REG_MEM:
4550	case PACKET3_MEM_WRITE:
4551	case PACKET3_PFP_SYNC_ME:
4552	case PACKET3_SURFACE_SYNC:
4553	case PACKET3_EVENT_WRITE:
4554	case PACKET3_EVENT_WRITE_EOP:
4555	case PACKET3_EVENT_WRITE_EOS:
4556	case PACKET3_SET_CONTEXT_REG:
4557	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4558	case PACKET3_SET_SH_REG:
4559	case PACKET3_SET_SH_REG_OFFSET:
4560	case PACKET3_INCREMENT_DE_COUNTER:
4561	case PACKET3_WAIT_ON_CE_COUNTER:
4562	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4563	case PACKET3_ME_WRITE:
4564		break;
4565	case PACKET3_COPY_DATA:
4566		if ((idx_value & 0xf00) == 0) {
4567			reg = ib[idx + 3] * 4;
4568			if (!si_vm_reg_valid(reg))
4569				return -EINVAL;
4570		}
4571		break;
4572	case PACKET3_WRITE_DATA:
4573		if ((idx_value & 0xf00) == 0) {
4574			start_reg = ib[idx + 1] * 4;
4575			if (idx_value & 0x10000) {
4576				if (!si_vm_reg_valid(start_reg))
4577					return -EINVAL;
4578			} else {
4579				for (i = 0; i < (pkt->count - 2); i++) {
4580					reg = start_reg + (4 * i);
4581					if (!si_vm_reg_valid(reg))
4582						return -EINVAL;
4583				}
4584			}
4585		}
4586		break;
4587	case PACKET3_COND_WRITE:
4588		if (idx_value & 0x100) {
4589			reg = ib[idx + 5] * 4;
4590			if (!si_vm_reg_valid(reg))
4591				return -EINVAL;
4592		}
4593		break;
4594	case PACKET3_COPY_DW:
4595		if (idx_value & 0x2) {
4596			reg = ib[idx + 3] * 4;
4597			if (!si_vm_reg_valid(reg))
4598				return -EINVAL;
4599		}
4600		break;
4601	case PACKET3_SET_CONFIG_REG:
4602		start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4603		end_reg = 4 * pkt->count + start_reg - 4;
4604		if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4605		    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4606		    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4607			DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4608			return -EINVAL;
4609		}
4610		for (i = 0; i < pkt->count; i++) {
4611			reg = start_reg + (4 * i);
4612			if (!si_vm_reg_valid(reg))
4613				return -EINVAL;
4614		}
4615		break;
4616	case PACKET3_CP_DMA:
4617		r = si_vm_packet3_cp_dma_check(ib, idx);
4618		if (r)
4619			return r;
4620		break;
4621	default:
4622		DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4623		return -EINVAL;
4624	}
4625	return 0;
4626}
4627
4628static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4629				       u32 *ib, struct radeon_cs_packet *pkt)
4630{
4631	int r;
4632	u32 idx = pkt->idx + 1;
4633	u32 idx_value = ib[idx];
4634	u32 start_reg, reg, i;
4635
4636	switch (pkt->opcode) {
4637	case PACKET3_NOP:
4638	case PACKET3_SET_BASE:
4639	case PACKET3_CLEAR_STATE:
4640	case PACKET3_DISPATCH_DIRECT:
4641	case PACKET3_DISPATCH_INDIRECT:
4642	case PACKET3_ALLOC_GDS:
4643	case PACKET3_WRITE_GDS_RAM:
4644	case PACKET3_ATOMIC_GDS:
4645	case PACKET3_ATOMIC:
4646	case PACKET3_OCCLUSION_QUERY:
4647	case PACKET3_SET_PREDICATION:
4648	case PACKET3_COND_EXEC:
4649	case PACKET3_PRED_EXEC:
4650	case PACKET3_CONTEXT_CONTROL:
4651	case PACKET3_STRMOUT_BUFFER_UPDATE:
4652	case PACKET3_WAIT_REG_MEM:
4653	case PACKET3_MEM_WRITE:
4654	case PACKET3_PFP_SYNC_ME:
4655	case PACKET3_SURFACE_SYNC:
4656	case PACKET3_EVENT_WRITE:
4657	case PACKET3_EVENT_WRITE_EOP:
4658	case PACKET3_EVENT_WRITE_EOS:
4659	case PACKET3_SET_CONTEXT_REG:
4660	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4661	case PACKET3_SET_SH_REG:
4662	case PACKET3_SET_SH_REG_OFFSET:
4663	case PACKET3_INCREMENT_DE_COUNTER:
4664	case PACKET3_WAIT_ON_CE_COUNTER:
4665	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4666	case PACKET3_ME_WRITE:
4667		break;
4668	case PACKET3_COPY_DATA:
4669		if ((idx_value & 0xf00) == 0) {
4670			reg = ib[idx + 3] * 4;
4671			if (!si_vm_reg_valid(reg))
4672				return -EINVAL;
4673		}
4674		break;
4675	case PACKET3_WRITE_DATA:
4676		if ((idx_value & 0xf00) == 0) {
4677			start_reg = ib[idx + 1] * 4;
4678			if (idx_value & 0x10000) {
4679				if (!si_vm_reg_valid(start_reg))
4680					return -EINVAL;
4681			} else {
4682				for (i = 0; i < (pkt->count - 2); i++) {
4683					reg = start_reg + (4 * i);
4684					if (!si_vm_reg_valid(reg))
4685						return -EINVAL;
4686				}
4687			}
4688		}
4689		break;
4690	case PACKET3_COND_WRITE:
4691		if (idx_value & 0x100) {
4692			reg = ib[idx + 5] * 4;
4693			if (!si_vm_reg_valid(reg))
4694				return -EINVAL;
4695		}
4696		break;
4697	case PACKET3_COPY_DW:
4698		if (idx_value & 0x2) {
4699			reg = ib[idx + 3] * 4;
4700			if (!si_vm_reg_valid(reg))
4701				return -EINVAL;
4702		}
4703		break;
4704	case PACKET3_CP_DMA:
4705		r = si_vm_packet3_cp_dma_check(ib, idx);
4706		if (r)
4707			return r;
4708		break;
4709	default:
4710		DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4711		return -EINVAL;
4712	}
4713	return 0;
4714}
4715
4716int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4717{
4718	int ret = 0;
4719	u32 idx = 0, i;
4720	struct radeon_cs_packet pkt;
4721
4722	do {
4723		pkt.idx = idx;
4724		pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4725		pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4726		pkt.one_reg_wr = 0;
4727		switch (pkt.type) {
4728		case RADEON_PACKET_TYPE0:
4729			dev_err(rdev->dev, "Packet0 not allowed!\n");
4730			ret = -EINVAL;
4731			break;
4732		case RADEON_PACKET_TYPE2:
4733			idx += 1;
4734			break;
4735		case RADEON_PACKET_TYPE3:
4736			pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4737			if (ib->is_const_ib)
4738				ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4739			else {
4740				switch (ib->ring) {
4741				case RADEON_RING_TYPE_GFX_INDEX:
4742					ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4743					break;
4744				case CAYMAN_RING_TYPE_CP1_INDEX:
4745				case CAYMAN_RING_TYPE_CP2_INDEX:
4746					ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4747					break;
4748				default:
4749					dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4750					ret = -EINVAL;
4751					break;
4752				}
4753			}
4754			idx += pkt.count + 2;
4755			break;
4756		default:
4757			dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4758			ret = -EINVAL;
4759			break;
4760		}
4761		if (ret) {
4762			for (i = 0; i < ib->length_dw; i++) {
4763				if (i == idx)
4764					printk("\t0x%08x <---\n", ib->ptr[i]);
4765				else
4766					printk("\t0x%08x\n", ib->ptr[i]);
4767			}
4768			break;
4769		}
4770	} while (idx < ib->length_dw);
4771
4772	return ret;
4773}
4774
4775/*
4776 * vm
4777 */
4778int si_vm_init(struct radeon_device *rdev)
4779{
4780	/* number of VMs */
4781	rdev->vm_manager.nvm = 16;
4782	/* base offset of vram pages */
4783	rdev->vm_manager.vram_base_offset = 0;
4784
4785	return 0;
4786}
4787
4788void si_vm_fini(struct radeon_device *rdev)
4789{
4790}
4791
4792/**
4793 * si_vm_decode_fault - print human readable fault info
4794 *
4795 * @rdev: radeon_device pointer
4796 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4797 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4798 *
4799 * Print human readable fault information (SI).
4800 */
4801static void si_vm_decode_fault(struct radeon_device *rdev,
4802			       u32 status, u32 addr)
4803{
4804	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4805	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4806	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4807	char *block;
4808
4809	if (rdev->family == CHIP_TAHITI) {
4810		switch (mc_id) {
4811		case 160:
4812		case 144:
4813		case 96:
4814		case 80:
4815		case 224:
4816		case 208:
4817		case 32:
4818		case 16:
4819			block = "CB";
4820			break;
4821		case 161:
4822		case 145:
4823		case 97:
4824		case 81:
4825		case 225:
4826		case 209:
4827		case 33:
4828		case 17:
4829			block = "CB_FMASK";
4830			break;
4831		case 162:
4832		case 146:
4833		case 98:
4834		case 82:
4835		case 226:
4836		case 210:
4837		case 34:
4838		case 18:
4839			block = "CB_CMASK";
4840			break;
4841		case 163:
4842		case 147:
4843		case 99:
4844		case 83:
4845		case 227:
4846		case 211:
4847		case 35:
4848		case 19:
4849			block = "CB_IMMED";
4850			break;
4851		case 164:
4852		case 148:
4853		case 100:
4854		case 84:
4855		case 228:
4856		case 212:
4857		case 36:
4858		case 20:
4859			block = "DB";
4860			break;
4861		case 165:
4862		case 149:
4863		case 101:
4864		case 85:
4865		case 229:
4866		case 213:
4867		case 37:
4868		case 21:
4869			block = "DB_HTILE";
4870			break;
4871		case 167:
4872		case 151:
4873		case 103:
4874		case 87:
4875		case 231:
4876		case 215:
4877		case 39:
4878		case 23:
4879			block = "DB_STEN";
4880			break;
4881		case 72:
4882		case 68:
4883		case 64:
4884		case 8:
4885		case 4:
4886		case 0:
4887		case 136:
4888		case 132:
4889		case 128:
4890		case 200:
4891		case 196:
4892		case 192:
4893			block = "TC";
4894			break;
4895		case 112:
4896		case 48:
4897			block = "CP";
4898			break;
4899		case 49:
4900		case 177:
4901		case 50:
4902		case 178:
4903			block = "SH";
4904			break;
4905		case 53:
4906		case 190:
4907			block = "VGT";
4908			break;
4909		case 117:
4910			block = "IH";
4911			break;
4912		case 51:
4913		case 115:
4914			block = "RLC";
4915			break;
4916		case 119:
4917		case 183:
4918			block = "DMA0";
4919			break;
4920		case 61:
4921			block = "DMA1";
4922			break;
4923		case 248:
4924		case 120:
4925			block = "HDP";
4926			break;
4927		default:
4928			block = "unknown";
4929			break;
4930		}
4931	} else {
4932		switch (mc_id) {
4933		case 32:
4934		case 16:
4935		case 96:
4936		case 80:
4937		case 160:
4938		case 144:
4939		case 224:
4940		case 208:
4941			block = "CB";
4942			break;
4943		case 33:
4944		case 17:
4945		case 97:
4946		case 81:
4947		case 161:
4948		case 145:
4949		case 225:
4950		case 209:
4951			block = "CB_FMASK";
4952			break;
4953		case 34:
4954		case 18:
4955		case 98:
4956		case 82:
4957		case 162:
4958		case 146:
4959		case 226:
4960		case 210:
4961			block = "CB_CMASK";
4962			break;
4963		case 35:
4964		case 19:
4965		case 99:
4966		case 83:
4967		case 163:
4968		case 147:
4969		case 227:
4970		case 211:
4971			block = "CB_IMMED";
4972			break;
4973		case 36:
4974		case 20:
4975		case 100:
4976		case 84:
4977		case 164:
4978		case 148:
4979		case 228:
4980		case 212:
4981			block = "DB";
4982			break;
4983		case 37:
4984		case 21:
4985		case 101:
4986		case 85:
4987		case 165:
4988		case 149:
4989		case 229:
4990		case 213:
4991			block = "DB_HTILE";
4992			break;
4993		case 39:
4994		case 23:
4995		case 103:
4996		case 87:
4997		case 167:
4998		case 151:
4999		case 231:
5000		case 215:
5001			block = "DB_STEN";
5002			break;
5003		case 72:
5004		case 68:
5005		case 8:
5006		case 4:
5007		case 136:
5008		case 132:
5009		case 200:
5010		case 196:
5011			block = "TC";
5012			break;
5013		case 112:
5014		case 48:
5015			block = "CP";
5016			break;
5017		case 49:
5018		case 177:
5019		case 50:
5020		case 178:
5021			block = "SH";
5022			break;
5023		case 53:
5024			block = "VGT";
5025			break;
5026		case 117:
5027			block = "IH";
5028			break;
5029		case 51:
5030		case 115:
5031			block = "RLC";
5032			break;
5033		case 119:
5034		case 183:
5035			block = "DMA0";
5036			break;
5037		case 61:
5038			block = "DMA1";
5039			break;
5040		case 248:
5041		case 120:
5042			block = "HDP";
5043			break;
5044		default:
5045			block = "unknown";
5046			break;
5047		}
5048	}
5049
5050	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5051	       protections, vmid, addr,
5052	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5053	       block, mc_id);
5054}
5055
5056void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5057		 unsigned vm_id, uint64_t pd_addr)
5058{
5059	/* write new base address */
5060	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5061	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5062				 WRITE_DATA_DST_SEL(0)));
5063
5064	if (vm_id < 8) {
5065		radeon_ring_write(ring,
5066				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5067	} else {
5068		radeon_ring_write(ring,
5069				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5070	}
5071	radeon_ring_write(ring, 0);
5072	radeon_ring_write(ring, pd_addr >> 12);
5073
5074	/* flush hdp cache */
5075	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5076	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5077				 WRITE_DATA_DST_SEL(0)));
5078	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5079	radeon_ring_write(ring, 0);
5080	radeon_ring_write(ring, 0x1);
5081
5082	/* bits 0-15 are the VM contexts0-15 */
5083	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5084	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5085				 WRITE_DATA_DST_SEL(0)));
5086	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5087	radeon_ring_write(ring, 0);
5088	radeon_ring_write(ring, 1 << vm_id);
5089
5090	/* wait for the invalidate to complete */
5091	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5092	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5093				 WAIT_REG_MEM_ENGINE(0))); /* me */
5094	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5095	radeon_ring_write(ring, 0);
5096	radeon_ring_write(ring, 0); /* ref */
5097	radeon_ring_write(ring, 0); /* mask */
5098	radeon_ring_write(ring, 0x20); /* poll interval */
5099
5100	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5101	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5102	radeon_ring_write(ring, 0x0);
5103}
5104
5105/*
5106 *  Power and clock gating
5107 */
5108static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5109{
5110	int i;
5111
5112	for (i = 0; i < rdev->usec_timeout; i++) {
5113		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5114			break;
5115		udelay(1);
5116	}
5117
5118	for (i = 0; i < rdev->usec_timeout; i++) {
5119		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5120			break;
5121		udelay(1);
5122	}
5123}
5124
5125static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5126					 bool enable)
5127{
5128	u32 tmp = RREG32(CP_INT_CNTL_RING0);
5129	u32 mask;
5130	int i;
5131
5132	if (enable)
5133		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5134	else
5135		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5136	WREG32(CP_INT_CNTL_RING0, tmp);
5137
5138	if (!enable) {
5139		/* read a gfx register */
5140		tmp = RREG32(DB_DEPTH_INFO);
5141
5142		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5143		for (i = 0; i < rdev->usec_timeout; i++) {
5144			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5145				break;
5146			udelay(1);
5147		}
5148	}
5149}
5150
5151static void si_set_uvd_dcm(struct radeon_device *rdev,
5152			   bool sw_mode)
5153{
5154	u32 tmp, tmp2;
5155
5156	tmp = RREG32(UVD_CGC_CTRL);
5157	tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5158	tmp |= DCM | CG_DT(1) | CLK_OD(4);
5159
5160	if (sw_mode) {
5161		tmp &= ~0x7ffff800;
5162		tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5163	} else {
5164		tmp |= 0x7ffff800;
5165		tmp2 = 0;
5166	}
5167
5168	WREG32(UVD_CGC_CTRL, tmp);
5169	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5170}
5171
5172void si_init_uvd_internal_cg(struct radeon_device *rdev)
5173{
5174	bool hw_mode = true;
5175
5176	if (hw_mode) {
5177		si_set_uvd_dcm(rdev, false);
5178	} else {
5179		u32 tmp = RREG32(UVD_CGC_CTRL);
5180		tmp &= ~DCM;
5181		WREG32(UVD_CGC_CTRL, tmp);
5182	}
5183}
5184
5185static u32 si_halt_rlc(struct radeon_device *rdev)
5186{
5187	u32 data, orig;
5188
5189	orig = data = RREG32(RLC_CNTL);
5190
5191	if (data & RLC_ENABLE) {
5192		data &= ~RLC_ENABLE;
5193		WREG32(RLC_CNTL, data);
5194
5195		si_wait_for_rlc_serdes(rdev);
5196	}
5197
5198	return orig;
5199}
5200
5201static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5202{
5203	u32 tmp;
5204
5205	tmp = RREG32(RLC_CNTL);
5206	if (tmp != rlc)
5207		WREG32(RLC_CNTL, rlc);
5208}
5209
5210static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5211{
5212	u32 data, orig;
5213
5214	orig = data = RREG32(DMA_PG);
5215	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5216		data |= PG_CNTL_ENABLE;
5217	else
5218		data &= ~PG_CNTL_ENABLE;
5219	if (orig != data)
5220		WREG32(DMA_PG, data);
5221}
5222
5223static void si_init_dma_pg(struct radeon_device *rdev)
5224{
5225	u32 tmp;
5226
5227	WREG32(DMA_PGFSM_WRITE,  0x00002000);
5228	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5229
5230	for (tmp = 0; tmp < 5; tmp++)
5231		WREG32(DMA_PGFSM_WRITE, 0);
5232}
5233
5234static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5235			       bool enable)
5236{
5237	u32 tmp;
5238
5239	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5240		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5241		WREG32(RLC_TTOP_D, tmp);
5242
5243		tmp = RREG32(RLC_PG_CNTL);
5244		tmp |= GFX_PG_ENABLE;
5245		WREG32(RLC_PG_CNTL, tmp);
5246
5247		tmp = RREG32(RLC_AUTO_PG_CTRL);
5248		tmp |= AUTO_PG_EN;
5249		WREG32(RLC_AUTO_PG_CTRL, tmp);
5250	} else {
5251		tmp = RREG32(RLC_AUTO_PG_CTRL);
5252		tmp &= ~AUTO_PG_EN;
5253		WREG32(RLC_AUTO_PG_CTRL, tmp);
5254
5255		tmp = RREG32(DB_RENDER_CONTROL);
5256	}
5257}
5258
5259static void si_init_gfx_cgpg(struct radeon_device *rdev)
5260{
5261	u32 tmp;
5262
5263	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5264
5265	tmp = RREG32(RLC_PG_CNTL);
5266	tmp |= GFX_PG_SRC;
5267	WREG32(RLC_PG_CNTL, tmp);
5268
5269	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5270
5271	tmp = RREG32(RLC_AUTO_PG_CTRL);
5272
5273	tmp &= ~GRBM_REG_SGIT_MASK;
5274	tmp |= GRBM_REG_SGIT(0x700);
5275	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5276	WREG32(RLC_AUTO_PG_CTRL, tmp);
5277}
5278
5279static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5280{
5281	u32 mask = 0, tmp, tmp1;
5282	int i;
5283
5284	si_select_se_sh(rdev, se, sh);
5285	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5286	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5287	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5288
5289	tmp &= 0xffff0000;
5290
5291	tmp |= tmp1;
5292	tmp >>= 16;
5293
5294	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5295		mask <<= 1;
5296		mask |= 1;
5297	}
5298
5299	return (~tmp) & mask;
5300}
5301
5302static void si_init_ao_cu_mask(struct radeon_device *rdev)
5303{
5304	u32 i, j, k, active_cu_number = 0;
5305	u32 mask, counter, cu_bitmap;
5306	u32 tmp = 0;
5307
5308	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5309		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5310			mask = 1;
5311			cu_bitmap = 0;
5312			counter  = 0;
5313			for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5314				if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5315					if (counter < 2)
5316						cu_bitmap |= mask;
5317					counter++;
5318				}
5319				mask <<= 1;
5320			}
5321
5322			active_cu_number += counter;
5323			tmp |= (cu_bitmap << (i * 16 + j * 8));
5324		}
5325	}
5326
5327	WREG32(RLC_PG_AO_CU_MASK, tmp);
5328
5329	tmp = RREG32(RLC_MAX_PG_CU);
5330	tmp &= ~MAX_PU_CU_MASK;
5331	tmp |= MAX_PU_CU(active_cu_number);
5332	WREG32(RLC_MAX_PG_CU, tmp);
5333}
5334
5335static void si_enable_cgcg(struct radeon_device *rdev,
5336			   bool enable)
5337{
5338	u32 data, orig, tmp;
5339
5340	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5341
5342	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5343		si_enable_gui_idle_interrupt(rdev, true);
5344
5345		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5346
5347		tmp = si_halt_rlc(rdev);
5348
5349		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5350		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5351		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5352
5353		si_wait_for_rlc_serdes(rdev);
5354
5355		si_update_rlc(rdev, tmp);
5356
5357		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5358
5359		data |= CGCG_EN | CGLS_EN;
5360	} else {
5361		si_enable_gui_idle_interrupt(rdev, false);
5362
5363		RREG32(CB_CGTT_SCLK_CTRL);
5364		RREG32(CB_CGTT_SCLK_CTRL);
5365		RREG32(CB_CGTT_SCLK_CTRL);
5366		RREG32(CB_CGTT_SCLK_CTRL);
5367
5368		data &= ~(CGCG_EN | CGLS_EN);
5369	}
5370
5371	if (orig != data)
5372		WREG32(RLC_CGCG_CGLS_CTRL, data);
5373}
5374
5375static void si_enable_mgcg(struct radeon_device *rdev,
5376			   bool enable)
5377{
5378	u32 data, orig, tmp = 0;
5379
5380	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5381		orig = data = RREG32(CGTS_SM_CTRL_REG);
5382		data = 0x96940200;
5383		if (orig != data)
5384			WREG32(CGTS_SM_CTRL_REG, data);
5385
5386		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5387			orig = data = RREG32(CP_MEM_SLP_CNTL);
5388			data |= CP_MEM_LS_EN;
5389			if (orig != data)
5390				WREG32(CP_MEM_SLP_CNTL, data);
5391		}
5392
5393		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5394		data &= 0xffffffc0;
5395		if (orig != data)
5396			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5397
5398		tmp = si_halt_rlc(rdev);
5399
5400		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5401		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5402		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5403
5404		si_update_rlc(rdev, tmp);
5405	} else {
5406		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5407		data |= 0x00000003;
5408		if (orig != data)
5409			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5410
5411		data = RREG32(CP_MEM_SLP_CNTL);
5412		if (data & CP_MEM_LS_EN) {
5413			data &= ~CP_MEM_LS_EN;
5414			WREG32(CP_MEM_SLP_CNTL, data);
5415		}
5416		orig = data = RREG32(CGTS_SM_CTRL_REG);
5417		data |= LS_OVERRIDE | OVERRIDE;
5418		if (orig != data)
5419			WREG32(CGTS_SM_CTRL_REG, data);
5420
5421		tmp = si_halt_rlc(rdev);
5422
5423		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5424		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5425		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5426
5427		si_update_rlc(rdev, tmp);
5428	}
5429}
5430
5431static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5432			       bool enable)
5433{
5434	u32 orig, data, tmp;
5435
5436	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5437		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5438		tmp |= 0x3fff;
5439		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5440
5441		orig = data = RREG32(UVD_CGC_CTRL);
5442		data |= DCM;
5443		if (orig != data)
5444			WREG32(UVD_CGC_CTRL, data);
5445
5446		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5447		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5448	} else {
5449		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5450		tmp &= ~0x3fff;
5451		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5452
5453		orig = data = RREG32(UVD_CGC_CTRL);
5454		data &= ~DCM;
5455		if (orig != data)
5456			WREG32(UVD_CGC_CTRL, data);
5457
5458		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5459		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5460	}
5461}
5462
5463static const u32 mc_cg_registers[] =
5464{
5465	MC_HUB_MISC_HUB_CG,
5466	MC_HUB_MISC_SIP_CG,
5467	MC_HUB_MISC_VM_CG,
5468	MC_XPB_CLK_GAT,
5469	ATC_MISC_CG,
5470	MC_CITF_MISC_WR_CG,
5471	MC_CITF_MISC_RD_CG,
5472	MC_CITF_MISC_VM_CG,
5473	VM_L2_CG,
5474};
5475
5476static void si_enable_mc_ls(struct radeon_device *rdev,
5477			    bool enable)
5478{
5479	int i;
5480	u32 orig, data;
5481
5482	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5483		orig = data = RREG32(mc_cg_registers[i]);
5484		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5485			data |= MC_LS_ENABLE;
5486		else
5487			data &= ~MC_LS_ENABLE;
5488		if (data != orig)
5489			WREG32(mc_cg_registers[i], data);
5490	}
5491}
5492
5493static void si_enable_mc_mgcg(struct radeon_device *rdev,
5494			       bool enable)
5495{
5496	int i;
5497	u32 orig, data;
5498
5499	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5500		orig = data = RREG32(mc_cg_registers[i]);
5501		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5502			data |= MC_CG_ENABLE;
5503		else
5504			data &= ~MC_CG_ENABLE;
5505		if (data != orig)
5506			WREG32(mc_cg_registers[i], data);
5507	}
5508}
5509
5510static void si_enable_dma_mgcg(struct radeon_device *rdev,
5511			       bool enable)
5512{
5513	u32 orig, data, offset;
5514	int i;
5515
5516	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5517		for (i = 0; i < 2; i++) {
5518			if (i == 0)
5519				offset = DMA0_REGISTER_OFFSET;
5520			else
5521				offset = DMA1_REGISTER_OFFSET;
5522			orig = data = RREG32(DMA_POWER_CNTL + offset);
5523			data &= ~MEM_POWER_OVERRIDE;
5524			if (data != orig)
5525				WREG32(DMA_POWER_CNTL + offset, data);
5526			WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5527		}
5528	} else {
5529		for (i = 0; i < 2; i++) {
5530			if (i == 0)
5531				offset = DMA0_REGISTER_OFFSET;
5532			else
5533				offset = DMA1_REGISTER_OFFSET;
5534			orig = data = RREG32(DMA_POWER_CNTL + offset);
5535			data |= MEM_POWER_OVERRIDE;
5536			if (data != orig)
5537				WREG32(DMA_POWER_CNTL + offset, data);
5538
5539			orig = data = RREG32(DMA_CLK_CTRL + offset);
5540			data = 0xff000000;
5541			if (data != orig)
5542				WREG32(DMA_CLK_CTRL + offset, data);
5543		}
5544	}
5545}
5546
5547static void si_enable_bif_mgls(struct radeon_device *rdev,
5548			       bool enable)
5549{
5550	u32 orig, data;
5551
5552	orig = data = RREG32_PCIE(PCIE_CNTL2);
5553
5554	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5555		data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5556			REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5557	else
5558		data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5559			  REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5560
5561	if (orig != data)
5562		WREG32_PCIE(PCIE_CNTL2, data);
5563}
5564
5565static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5566			       bool enable)
5567{
5568	u32 orig, data;
5569
5570	orig = data = RREG32(HDP_HOST_PATH_CNTL);
5571
5572	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5573		data &= ~CLOCK_GATING_DIS;
5574	else
5575		data |= CLOCK_GATING_DIS;
5576
5577	if (orig != data)
5578		WREG32(HDP_HOST_PATH_CNTL, data);
5579}
5580
5581static void si_enable_hdp_ls(struct radeon_device *rdev,
5582			     bool enable)
5583{
5584	u32 orig, data;
5585
5586	orig = data = RREG32(HDP_MEM_POWER_LS);
5587
5588	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5589		data |= HDP_LS_ENABLE;
5590	else
5591		data &= ~HDP_LS_ENABLE;
5592
5593	if (orig != data)
5594		WREG32(HDP_MEM_POWER_LS, data);
5595}
5596
5597static void si_update_cg(struct radeon_device *rdev,
5598			 u32 block, bool enable)
5599{
5600	if (block & RADEON_CG_BLOCK_GFX) {
5601		si_enable_gui_idle_interrupt(rdev, false);
5602		/* order matters! */
5603		if (enable) {
5604			si_enable_mgcg(rdev, true);
5605			si_enable_cgcg(rdev, true);
5606		} else {
5607			si_enable_cgcg(rdev, false);
5608			si_enable_mgcg(rdev, false);
5609		}
5610		si_enable_gui_idle_interrupt(rdev, true);
5611	}
5612
5613	if (block & RADEON_CG_BLOCK_MC) {
5614		si_enable_mc_mgcg(rdev, enable);
5615		si_enable_mc_ls(rdev, enable);
5616	}
5617
5618	if (block & RADEON_CG_BLOCK_SDMA) {
5619		si_enable_dma_mgcg(rdev, enable);
5620	}
5621
5622	if (block & RADEON_CG_BLOCK_BIF) {
5623		si_enable_bif_mgls(rdev, enable);
5624	}
5625
5626	if (block & RADEON_CG_BLOCK_UVD) {
5627		if (rdev->has_uvd) {
5628			si_enable_uvd_mgcg(rdev, enable);
5629		}
5630	}
5631
5632	if (block & RADEON_CG_BLOCK_HDP) {
5633		si_enable_hdp_mgcg(rdev, enable);
5634		si_enable_hdp_ls(rdev, enable);
5635	}
5636}
5637
5638static void si_init_cg(struct radeon_device *rdev)
5639{
5640	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5641			    RADEON_CG_BLOCK_MC |
5642			    RADEON_CG_BLOCK_SDMA |
5643			    RADEON_CG_BLOCK_BIF |
5644			    RADEON_CG_BLOCK_HDP), true);
5645	if (rdev->has_uvd) {
5646		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5647		si_init_uvd_internal_cg(rdev);
5648	}
5649}
5650
5651static void si_fini_cg(struct radeon_device *rdev)
5652{
5653	if (rdev->has_uvd) {
5654		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5655	}
5656	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5657			    RADEON_CG_BLOCK_MC |
5658			    RADEON_CG_BLOCK_SDMA |
5659			    RADEON_CG_BLOCK_BIF |
5660			    RADEON_CG_BLOCK_HDP), false);
5661}
5662
5663u32 si_get_csb_size(struct radeon_device *rdev)
5664{
5665	u32 count = 0;
5666	const struct cs_section_def *sect = NULL;
5667	const struct cs_extent_def *ext = NULL;
5668
5669	if (rdev->rlc.cs_data == NULL)
5670		return 0;
5671
5672	/* begin clear state */
5673	count += 2;
5674	/* context control state */
5675	count += 3;
5676
5677	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5678		for (ext = sect->section; ext->extent != NULL; ++ext) {
5679			if (sect->id == SECT_CONTEXT)
5680				count += 2 + ext->reg_count;
5681			else
5682				return 0;
5683		}
5684	}
5685	/* pa_sc_raster_config */
5686	count += 3;
5687	/* end clear state */
5688	count += 2;
5689	/* clear state */
5690	count += 2;
5691
5692	return count;
5693}
5694
5695void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5696{
5697	u32 count = 0, i;
5698	const struct cs_section_def *sect = NULL;
5699	const struct cs_extent_def *ext = NULL;
5700
5701	if (rdev->rlc.cs_data == NULL)
5702		return;
5703	if (buffer == NULL)
5704		return;
5705
5706	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5707	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5708
5709	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5710	buffer[count++] = cpu_to_le32(0x80000000);
5711	buffer[count++] = cpu_to_le32(0x80000000);
5712
5713	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5714		for (ext = sect->section; ext->extent != NULL; ++ext) {
5715			if (sect->id == SECT_CONTEXT) {
5716				buffer[count++] =
5717					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5718				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5719				for (i = 0; i < ext->reg_count; i++)
5720					buffer[count++] = cpu_to_le32(ext->extent[i]);
5721			} else {
5722				return;
5723			}
5724		}
5725	}
5726
5727	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5728	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5729	switch (rdev->family) {
5730	case CHIP_TAHITI:
5731	case CHIP_PITCAIRN:
5732		buffer[count++] = cpu_to_le32(0x2a00126a);
5733		break;
5734	case CHIP_VERDE:
5735		buffer[count++] = cpu_to_le32(0x0000124a);
5736		break;
5737	case CHIP_OLAND:
5738		buffer[count++] = cpu_to_le32(0x00000082);
5739		break;
5740	case CHIP_HAINAN:
5741		buffer[count++] = cpu_to_le32(0x00000000);
5742		break;
5743	default:
5744		buffer[count++] = cpu_to_le32(0x00000000);
5745		break;
5746	}
5747
5748	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5749	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5750
5751	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5752	buffer[count++] = cpu_to_le32(0);
5753}
5754
5755static void si_init_pg(struct radeon_device *rdev)
5756{
5757	if (rdev->pg_flags) {
5758		if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5759			si_init_dma_pg(rdev);
5760		}
5761		si_init_ao_cu_mask(rdev);
5762		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5763			si_init_gfx_cgpg(rdev);
5764		} else {
5765			WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5766			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5767		}
5768		si_enable_dma_pg(rdev, true);
5769		si_enable_gfx_cgpg(rdev, true);
5770	} else {
5771		WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5772		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5773	}
5774}
5775
5776static void si_fini_pg(struct radeon_device *rdev)
5777{
5778	if (rdev->pg_flags) {
5779		si_enable_dma_pg(rdev, false);
5780		si_enable_gfx_cgpg(rdev, false);
5781	}
5782}
5783
5784/*
5785 * RLC
5786 */
5787void si_rlc_reset(struct radeon_device *rdev)
5788{
5789	u32 tmp = RREG32(GRBM_SOFT_RESET);
5790
5791	tmp |= SOFT_RESET_RLC;
5792	WREG32(GRBM_SOFT_RESET, tmp);
5793	udelay(50);
5794	tmp &= ~SOFT_RESET_RLC;
5795	WREG32(GRBM_SOFT_RESET, tmp);
5796	udelay(50);
5797}
5798
5799static void si_rlc_stop(struct radeon_device *rdev)
5800{
5801	WREG32(RLC_CNTL, 0);
5802
5803	si_enable_gui_idle_interrupt(rdev, false);
5804
5805	si_wait_for_rlc_serdes(rdev);
5806}
5807
5808static void si_rlc_start(struct radeon_device *rdev)
5809{
5810	WREG32(RLC_CNTL, RLC_ENABLE);
5811
5812	si_enable_gui_idle_interrupt(rdev, true);
5813
5814	udelay(50);
5815}
5816
5817static bool si_lbpw_supported(struct radeon_device *rdev)
5818{
5819	u32 tmp;
5820
5821	/* Enable LBPW only for DDR3 */
5822	tmp = RREG32(MC_SEQ_MISC0);
5823	if ((tmp & 0xF0000000) == 0xB0000000)
5824		return true;
5825	return false;
5826}
5827
5828static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5829{
5830	u32 tmp;
5831
5832	tmp = RREG32(RLC_LB_CNTL);
5833	if (enable)
5834		tmp |= LOAD_BALANCE_ENABLE;
5835	else
5836		tmp &= ~LOAD_BALANCE_ENABLE;
5837	WREG32(RLC_LB_CNTL, tmp);
5838
5839	if (!enable) {
5840		si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5841		WREG32(SPI_LB_CU_MASK, 0x00ff);
5842	}
5843}
5844
5845static int si_rlc_resume(struct radeon_device *rdev)
5846{
5847	u32 i;
5848
5849	if (!rdev->rlc_fw)
5850		return -EINVAL;
5851
5852	si_rlc_stop(rdev);
5853
5854	si_rlc_reset(rdev);
5855
5856	si_init_pg(rdev);
5857
5858	si_init_cg(rdev);
5859
5860	WREG32(RLC_RL_BASE, 0);
5861	WREG32(RLC_RL_SIZE, 0);
5862	WREG32(RLC_LB_CNTL, 0);
5863	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5864	WREG32(RLC_LB_CNTR_INIT, 0);
5865	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5866
5867	WREG32(RLC_MC_CNTL, 0);
5868	WREG32(RLC_UCODE_CNTL, 0);
5869
5870	if (rdev->new_fw) {
5871		const struct rlc_firmware_header_v1_0 *hdr =
5872			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5873		u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5874		const __le32 *fw_data = (const __le32 *)
5875			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5876
5877		radeon_ucode_print_rlc_hdr(&hdr->header);
5878
5879		for (i = 0; i < fw_size; i++) {
5880			WREG32(RLC_UCODE_ADDR, i);
5881			WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5882		}
5883	} else {
5884		const __be32 *fw_data =
5885			(const __be32 *)rdev->rlc_fw->data;
5886		for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5887			WREG32(RLC_UCODE_ADDR, i);
5888			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5889		}
5890	}
5891	WREG32(RLC_UCODE_ADDR, 0);
5892
5893	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5894
5895	si_rlc_start(rdev);
5896
5897	return 0;
5898}
5899
5900static void si_enable_interrupts(struct radeon_device *rdev)
5901{
5902	u32 ih_cntl = RREG32(IH_CNTL);
5903	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5904
5905	ih_cntl |= ENABLE_INTR;
5906	ih_rb_cntl |= IH_RB_ENABLE;
5907	WREG32(IH_CNTL, ih_cntl);
5908	WREG32(IH_RB_CNTL, ih_rb_cntl);
5909	rdev->ih.enabled = true;
5910}
5911
5912static void si_disable_interrupts(struct radeon_device *rdev)
5913{
5914	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5915	u32 ih_cntl = RREG32(IH_CNTL);
5916
5917	ih_rb_cntl &= ~IH_RB_ENABLE;
5918	ih_cntl &= ~ENABLE_INTR;
5919	WREG32(IH_RB_CNTL, ih_rb_cntl);
5920	WREG32(IH_CNTL, ih_cntl);
5921	/* set rptr, wptr to 0 */
5922	WREG32(IH_RB_RPTR, 0);
5923	WREG32(IH_RB_WPTR, 0);
5924	rdev->ih.enabled = false;
5925	rdev->ih.rptr = 0;
5926}
5927
5928static void si_disable_interrupt_state(struct radeon_device *rdev)
5929{
5930	int i;
5931	u32 tmp;
5932
5933	tmp = RREG32(CP_INT_CNTL_RING0) &
5934		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5935	WREG32(CP_INT_CNTL_RING0, tmp);
5936	WREG32(CP_INT_CNTL_RING1, 0);
5937	WREG32(CP_INT_CNTL_RING2, 0);
5938	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5939	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5940	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5941	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5942	WREG32(GRBM_INT_CNTL, 0);
5943	WREG32(SRBM_INT_CNTL, 0);
5944	for (i = 0; i < rdev->num_crtc; i++)
5945		WREG32(INT_MASK + crtc_offsets[i], 0);
5946	for (i = 0; i < rdev->num_crtc; i++)
5947		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
5948
5949	if (!ASIC_IS_NODCE(rdev)) {
5950		WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5951
5952		for (i = 0; i < 6; i++)
5953			WREG32_AND(DC_HPDx_INT_CONTROL(i),
5954				   DC_HPDx_INT_POLARITY);
5955	}
5956}
5957
5958static int si_irq_init(struct radeon_device *rdev)
5959{
5960	int ret = 0;
5961	int rb_bufsz;
5962	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5963
5964	/* allocate ring */
5965	ret = r600_ih_ring_alloc(rdev);
5966	if (ret)
5967		return ret;
5968
5969	/* disable irqs */
5970	si_disable_interrupts(rdev);
5971
5972	/* init rlc */
5973	ret = si_rlc_resume(rdev);
5974	if (ret) {
5975		r600_ih_ring_fini(rdev);
5976		return ret;
5977	}
5978
5979	/* setup interrupt control */
5980	/* set dummy read address to dummy page address */
5981	WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
5982	interrupt_cntl = RREG32(INTERRUPT_CNTL);
5983	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5984	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5985	 */
5986	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5987	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5988	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5989	WREG32(INTERRUPT_CNTL, interrupt_cntl);
5990
5991	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5992	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5993
5994	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5995		      IH_WPTR_OVERFLOW_CLEAR |
5996		      (rb_bufsz << 1));
5997
5998	if (rdev->wb.enabled)
5999		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6000
6001	/* set the writeback address whether it's enabled or not */
6002	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6003	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6004
6005	WREG32(IH_RB_CNTL, ih_rb_cntl);
6006
6007	/* set rptr, wptr to 0 */
6008	WREG32(IH_RB_RPTR, 0);
6009	WREG32(IH_RB_WPTR, 0);
6010
6011	/* Default settings for IH_CNTL (disabled at first) */
6012	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6013	/* RPTR_REARM only works if msi's are enabled */
6014	if (rdev->msi_enabled)
6015		ih_cntl |= RPTR_REARM;
6016	WREG32(IH_CNTL, ih_cntl);
6017
6018	/* force the active interrupt state to all disabled */
6019	si_disable_interrupt_state(rdev);
6020
6021	pci_set_master(rdev->pdev);
6022
6023	/* enable irqs */
6024	si_enable_interrupts(rdev);
6025
6026	return ret;
6027}
6028
6029/* The order we write back each register here is important */
6030int si_irq_set(struct radeon_device *rdev)
6031{
6032	int i;
6033	u32 cp_int_cntl;
6034	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6035	u32 grbm_int_cntl = 0;
6036	u32 dma_cntl, dma_cntl1;
6037	u32 thermal_int = 0;
6038
6039	if (!rdev->irq.installed) {
6040		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6041		return -EINVAL;
6042	}
6043	/* don't enable anything if the ih is disabled */
6044	if (!rdev->ih.enabled) {
6045		si_disable_interrupts(rdev);
6046		/* force the active interrupt state to all disabled */
6047		si_disable_interrupt_state(rdev);
6048		return 0;
6049	}
6050
6051	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6052		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6053
6054	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6055	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6056
6057	thermal_int = RREG32(CG_THERMAL_INT) &
6058		~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6059
6060	/* enable CP interrupts on all rings */
6061	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6062		DRM_DEBUG("si_irq_set: sw int gfx\n");
6063		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6064	}
6065	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6066		DRM_DEBUG("si_irq_set: sw int cp1\n");
6067		cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6068	}
6069	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6070		DRM_DEBUG("si_irq_set: sw int cp2\n");
6071		cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6072	}
6073	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6074		DRM_DEBUG("si_irq_set: sw int dma\n");
6075		dma_cntl |= TRAP_ENABLE;
6076	}
6077
6078	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6079		DRM_DEBUG("si_irq_set: sw int dma1\n");
6080		dma_cntl1 |= TRAP_ENABLE;
6081	}
6082
6083	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6084	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6085	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6086
6087	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6088	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6089
6090	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6091
6092	if (rdev->irq.dpm_thermal) {
6093		DRM_DEBUG("dpm thermal\n");
6094		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6095	}
6096
6097	for (i = 0; i < rdev->num_crtc; i++) {
6098		radeon_irq_kms_set_irq_n_enabled(
6099		    rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6100		    rdev->irq.crtc_vblank_int[i] ||
6101		    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
6102	}
6103
6104	for (i = 0; i < rdev->num_crtc; i++)
6105		WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
6106
6107	if (!ASIC_IS_NODCE(rdev)) {
6108		for (i = 0; i < 6; i++) {
6109			radeon_irq_kms_set_irq_n_enabled(
6110			    rdev, DC_HPDx_INT_CONTROL(i),
6111			    DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6112			    rdev->irq.hpd[i], "HPD", i);
6113		}
6114	}
6115
6116	WREG32(CG_THERMAL_INT, thermal_int);
6117
6118	/* posting read */
6119	RREG32(SRBM_STATUS);
6120
6121	return 0;
6122}
6123
6124/* The order we write back each register here is important */
6125static inline void si_irq_ack(struct radeon_device *rdev)
6126{
6127	int i, j;
6128	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6129	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
6130
6131	if (ASIC_IS_NODCE(rdev))
6132		return;
6133
6134	for (i = 0; i < 6; i++) {
6135		disp_int[i] = RREG32(si_disp_int_status[i]);
6136		if (i < rdev->num_crtc)
6137			grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
6138	}
6139
6140	/* We write back each interrupt register in pairs of two */
6141	for (i = 0; i < rdev->num_crtc; i += 2) {
6142		for (j = i; j < (i + 2); j++) {
6143			if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6144				WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6145				       GRPH_PFLIP_INT_CLEAR);
6146		}
6147
6148		for (j = i; j < (i + 2); j++) {
6149			if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6150				WREG32(VBLANK_STATUS + crtc_offsets[j],
6151				       VBLANK_ACK);
6152			if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6153				WREG32(VLINE_STATUS + crtc_offsets[j],
6154				       VLINE_ACK);
6155		}
6156	}
6157
6158	for (i = 0; i < 6; i++) {
6159		if (disp_int[i] & DC_HPD1_INTERRUPT)
6160			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6161	}
6162
6163	for (i = 0; i < 6; i++) {
6164		if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6165			WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
6166	}
6167}
6168
6169static void si_irq_disable(struct radeon_device *rdev)
6170{
6171	si_disable_interrupts(rdev);
6172	/* Wait and acknowledge irq */
6173	mdelay(1);
6174	si_irq_ack(rdev);
6175	si_disable_interrupt_state(rdev);
6176}
6177
6178static void si_irq_suspend(struct radeon_device *rdev)
6179{
6180	si_irq_disable(rdev);
6181	si_rlc_stop(rdev);
6182}
6183
6184static void si_irq_fini(struct radeon_device *rdev)
6185{
6186	si_irq_suspend(rdev);
6187	r600_ih_ring_fini(rdev);
6188}
6189
6190static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6191{
6192	u32 wptr, tmp;
6193
6194	if (rdev->wb.enabled)
6195		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6196	else
6197		wptr = RREG32(IH_RB_WPTR);
6198
6199	if (wptr & RB_OVERFLOW) {
6200		wptr &= ~RB_OVERFLOW;
6201		/* When a ring buffer overflow happen start parsing interrupt
6202		 * from the last not overwritten vector (wptr + 16). Hopefully
6203		 * this should allow us to catchup.
6204		 */
6205		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6206			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6207		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6208		tmp = RREG32(IH_RB_CNTL);
6209		tmp |= IH_WPTR_OVERFLOW_CLEAR;
6210		WREG32(IH_RB_CNTL, tmp);
6211	}
6212	return (wptr & rdev->ih.ptr_mask);
6213}
6214
6215/*        SI IV Ring
6216 * Each IV ring entry is 128 bits:
6217 * [7:0]    - interrupt source id
6218 * [31:8]   - reserved
6219 * [59:32]  - interrupt source data
6220 * [63:60]  - reserved
6221 * [71:64]  - RINGID
6222 * [79:72]  - VMID
6223 * [127:80] - reserved
6224 */
6225int si_irq_process(struct radeon_device *rdev)
6226{
6227	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6228	u32 crtc_idx, hpd_idx;
6229	u32 mask;
6230	u32 wptr;
6231	u32 rptr;
6232	u32 src_id, src_data, ring_id;
6233	u32 ring_index;
6234	bool queue_hotplug = false;
6235	bool queue_dp = false;
6236	bool queue_thermal = false;
6237	u32 status, addr;
6238	const char *event_name;
6239
6240	if (!rdev->ih.enabled || rdev->shutdown)
6241		return IRQ_NONE;
6242
6243	wptr = si_get_ih_wptr(rdev);
6244
6245restart_ih:
6246	/* is somebody else already processing irqs? */
6247	if (atomic_xchg(&rdev->ih.lock, 1))
6248		return IRQ_NONE;
6249
6250	rptr = rdev->ih.rptr;
6251	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6252
6253	/* Order reading of wptr vs. reading of IH ring data */
6254	rmb();
6255
6256	/* display interrupts */
6257	si_irq_ack(rdev);
6258
6259	while (rptr != wptr) {
6260		/* wptr/rptr are in bytes! */
6261		ring_index = rptr / 4;
6262		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6263		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6264		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6265
6266		switch (src_id) {
6267		case 1: /* D1 vblank/vline */
6268		case 2: /* D2 vblank/vline */
6269		case 3: /* D3 vblank/vline */
6270		case 4: /* D4 vblank/vline */
6271		case 5: /* D5 vblank/vline */
6272		case 6: /* D6 vblank/vline */
6273			crtc_idx = src_id - 1;
6274
6275			if (src_data == 0) { /* vblank */
6276				mask = LB_D1_VBLANK_INTERRUPT;
6277				event_name = "vblank";
6278
6279				if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6280					drm_handle_vblank(rdev_to_drm(rdev), crtc_idx);
6281					rdev->pm.vblank_sync = true;
6282					wake_up(&rdev->irq.vblank_queue);
6283				}
6284				if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6285					radeon_crtc_handle_vblank(rdev,
6286								  crtc_idx);
6287				}
6288
6289			} else if (src_data == 1) { /* vline */
6290				mask = LB_D1_VLINE_INTERRUPT;
6291				event_name = "vline";
6292			} else {
6293				DRM_DEBUG("Unhandled interrupt: %d %d\n",
6294					  src_id, src_data);
6295				break;
6296			}
6297
6298			if (!(disp_int[crtc_idx] & mask)) {
6299				DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6300					  crtc_idx + 1, event_name);
6301			}
6302
6303			disp_int[crtc_idx] &= ~mask;
6304			DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
6305
6306			break;
6307		case 8: /* D1 page flip */
6308		case 10: /* D2 page flip */
6309		case 12: /* D3 page flip */
6310		case 14: /* D4 page flip */
6311		case 16: /* D5 page flip */
6312		case 18: /* D6 page flip */
6313			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6314			if (radeon_use_pflipirq > 0)
6315				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6316			break;
6317		case 42: /* HPD hotplug */
6318			if (src_data <= 5) {
6319				hpd_idx = src_data;
6320				mask = DC_HPD1_INTERRUPT;
6321				queue_hotplug = true;
6322				event_name = "HPD";
6323
6324			} else if (src_data <= 11) {
6325				hpd_idx = src_data - 6;
6326				mask = DC_HPD1_RX_INTERRUPT;
6327				queue_dp = true;
6328				event_name = "HPD_RX";
6329
6330			} else {
6331				DRM_DEBUG("Unhandled interrupt: %d %d\n",
6332					  src_id, src_data);
6333				break;
6334			}
6335
6336			if (!(disp_int[hpd_idx] & mask))
6337				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6338
6339			disp_int[hpd_idx] &= ~mask;
6340			DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
6341			break;
6342		case 96:
6343			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6344			WREG32(SRBM_INT_ACK, 0x1);
6345			break;
6346		case 124: /* UVD */
6347			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6348			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6349			break;
6350		case 146:
6351		case 147:
6352			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6353			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6354			/* reset addr and status */
6355			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6356			if (addr == 0x0 && status == 0x0)
6357				break;
6358			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6359			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6360				addr);
6361			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6362				status);
6363			si_vm_decode_fault(rdev, status, addr);
6364			break;
6365		case 176: /* RINGID0 CP_INT */
6366			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6367			break;
6368		case 177: /* RINGID1 CP_INT */
6369			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6370			break;
6371		case 178: /* RINGID2 CP_INT */
6372			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6373			break;
6374		case 181: /* CP EOP event */
6375			DRM_DEBUG("IH: CP EOP\n");
6376			switch (ring_id) {
6377			case 0:
6378				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6379				break;
6380			case 1:
6381				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6382				break;
6383			case 2:
6384				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6385				break;
6386			}
6387			break;
6388		case 224: /* DMA trap event */
6389			DRM_DEBUG("IH: DMA trap\n");
6390			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6391			break;
6392		case 230: /* thermal low to high */
6393			DRM_DEBUG("IH: thermal low to high\n");
6394			rdev->pm.dpm.thermal.high_to_low = false;
6395			queue_thermal = true;
6396			break;
6397		case 231: /* thermal high to low */
6398			DRM_DEBUG("IH: thermal high to low\n");
6399			rdev->pm.dpm.thermal.high_to_low = true;
6400			queue_thermal = true;
6401			break;
6402		case 233: /* GUI IDLE */
6403			DRM_DEBUG("IH: GUI idle\n");
6404			break;
6405		case 244: /* DMA trap event */
6406			DRM_DEBUG("IH: DMA1 trap\n");
6407			radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6408			break;
6409		default:
6410			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6411			break;
6412		}
6413
6414		/* wptr/rptr are in bytes! */
6415		rptr += 16;
6416		rptr &= rdev->ih.ptr_mask;
6417		WREG32(IH_RB_RPTR, rptr);
6418	}
6419	if (queue_dp)
6420		schedule_work(&rdev->dp_work);
6421	if (queue_hotplug)
6422		schedule_delayed_work(&rdev->hotplug_work, 0);
6423	if (queue_thermal && rdev->pm.dpm_enabled)
6424		schedule_work(&rdev->pm.dpm.thermal.work);
6425	rdev->ih.rptr = rptr;
6426	atomic_set(&rdev->ih.lock, 0);
6427
6428	/* make sure wptr hasn't changed while processing */
6429	wptr = si_get_ih_wptr(rdev);
6430	if (wptr != rptr)
6431		goto restart_ih;
6432
6433	return IRQ_HANDLED;
6434}
6435
6436/*
6437 * startup/shutdown callbacks
6438 */
6439static void si_uvd_init(struct radeon_device *rdev)
6440{
6441	int r;
6442
6443	if (!rdev->has_uvd)
6444		return;
6445
6446	r = radeon_uvd_init(rdev);
6447	if (r) {
6448		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6449		/*
6450		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6451		 * to early fails uvd_v2_2_resume() and thus nothing happens
6452		 * there. So it is pointless to try to go through that code
6453		 * hence why we disable uvd here.
6454		 */
6455		rdev->has_uvd = false;
6456		return;
6457	}
6458	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6459	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6460}
6461
6462static void si_uvd_start(struct radeon_device *rdev)
6463{
6464	int r;
6465
6466	if (!rdev->has_uvd)
6467		return;
6468
6469	r = uvd_v2_2_resume(rdev);
6470	if (r) {
6471		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6472		goto error;
6473	}
6474	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6475	if (r) {
6476		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6477		goto error;
6478	}
6479	return;
6480
6481error:
6482	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6483}
6484
6485static void si_uvd_resume(struct radeon_device *rdev)
6486{
6487	struct radeon_ring *ring;
6488	int r;
6489
6490	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6491		return;
6492
6493	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6494	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6495	if (r) {
6496		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6497		return;
6498	}
6499	r = uvd_v1_0_init(rdev);
6500	if (r) {
6501		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6502		return;
6503	}
6504}
6505
6506static void si_vce_init(struct radeon_device *rdev)
6507{
6508	int r;
6509
6510	if (!rdev->has_vce)
6511		return;
6512
6513	r = radeon_vce_init(rdev);
6514	if (r) {
6515		dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6516		/*
6517		 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6518		 * to early fails si_vce_start() and thus nothing happens
6519		 * there. So it is pointless to try to go through that code
6520		 * hence why we disable vce here.
6521		 */
6522		rdev->has_vce = false;
6523		return;
6524	}
6525	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6526	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6527	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6528	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6529}
6530
6531static void si_vce_start(struct radeon_device *rdev)
6532{
6533	int r;
6534
6535	if (!rdev->has_vce)
6536		return;
6537
6538	r = radeon_vce_resume(rdev);
6539	if (r) {
6540		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6541		goto error;
6542	}
6543	r = vce_v1_0_resume(rdev);
6544	if (r) {
6545		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6546		goto error;
6547	}
6548	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6549	if (r) {
6550		dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6551		goto error;
6552	}
6553	r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6554	if (r) {
6555		dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6556		goto error;
6557	}
6558	return;
6559
6560error:
6561	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6562	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6563}
6564
6565static void si_vce_resume(struct radeon_device *rdev)
6566{
6567	struct radeon_ring *ring;
6568	int r;
6569
6570	if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6571		return;
6572
6573	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6574	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6575	if (r) {
6576		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6577		return;
6578	}
6579	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6580	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6581	if (r) {
6582		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6583		return;
6584	}
6585	r = vce_v1_0_init(rdev);
6586	if (r) {
6587		dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6588		return;
6589	}
6590}
6591
6592static int si_startup(struct radeon_device *rdev)
6593{
6594	struct radeon_ring *ring;
6595	int r;
6596
6597	/* enable pcie gen2/3 link */
6598	si_pcie_gen3_enable(rdev);
6599	/* enable aspm */
6600	si_program_aspm(rdev);
6601
6602	/* scratch needs to be initialized before MC */
6603	r = r600_vram_scratch_init(rdev);
6604	if (r)
6605		return r;
6606
6607	si_mc_program(rdev);
6608
6609	if (!rdev->pm.dpm_enabled) {
6610		r = si_mc_load_microcode(rdev);
6611		if (r) {
6612			DRM_ERROR("Failed to load MC firmware!\n");
6613			return r;
6614		}
6615	}
6616
6617	r = si_pcie_gart_enable(rdev);
6618	if (r)
6619		return r;
6620	si_gpu_init(rdev);
6621
6622	/* allocate rlc buffers */
6623	if (rdev->family == CHIP_VERDE) {
6624		rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6625		rdev->rlc.reg_list_size =
6626			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6627	}
6628	rdev->rlc.cs_data = si_cs_data;
6629	r = sumo_rlc_init(rdev);
6630	if (r) {
6631		DRM_ERROR("Failed to init rlc BOs!\n");
6632		return r;
6633	}
6634
6635	/* allocate wb buffer */
6636	r = radeon_wb_init(rdev);
6637	if (r)
6638		return r;
6639
6640	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6641	if (r) {
6642		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6643		return r;
6644	}
6645
6646	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6647	if (r) {
6648		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6649		return r;
6650	}
6651
6652	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6653	if (r) {
6654		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6655		return r;
6656	}
6657
6658	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6659	if (r) {
6660		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6661		return r;
6662	}
6663
6664	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6665	if (r) {
6666		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6667		return r;
6668	}
6669
6670	si_uvd_start(rdev);
6671	si_vce_start(rdev);
6672
6673	/* Enable IRQ */
6674	if (!rdev->irq.installed) {
6675		r = radeon_irq_kms_init(rdev);
6676		if (r)
6677			return r;
6678	}
6679
6680	r = si_irq_init(rdev);
6681	if (r) {
6682		DRM_ERROR("radeon: IH init failed (%d).\n", r);
6683		radeon_irq_kms_fini(rdev);
6684		return r;
6685	}
6686	si_irq_set(rdev);
6687
6688	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6689	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6690			     RADEON_CP_PACKET2);
6691	if (r)
6692		return r;
6693
6694	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6695	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6696			     RADEON_CP_PACKET2);
6697	if (r)
6698		return r;
6699
6700	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6701	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6702			     RADEON_CP_PACKET2);
6703	if (r)
6704		return r;
6705
6706	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6707	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6708			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6709	if (r)
6710		return r;
6711
6712	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6713	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6714			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6715	if (r)
6716		return r;
6717
6718	r = si_cp_load_microcode(rdev);
6719	if (r)
6720		return r;
6721	r = si_cp_resume(rdev);
6722	if (r)
6723		return r;
6724
6725	r = cayman_dma_resume(rdev);
6726	if (r)
6727		return r;
6728
6729	si_uvd_resume(rdev);
6730	si_vce_resume(rdev);
6731
6732	r = radeon_ib_pool_init(rdev);
6733	if (r) {
6734		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6735		return r;
6736	}
6737
6738	r = radeon_vm_manager_init(rdev);
6739	if (r) {
6740		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6741		return r;
6742	}
6743
6744	r = radeon_audio_init(rdev);
6745	if (r)
6746		return r;
6747
6748	return 0;
6749}
6750
6751int si_resume(struct radeon_device *rdev)
6752{
6753	int r;
6754
6755	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6756	 * posting will perform necessary task to bring back GPU into good
6757	 * shape.
6758	 */
6759	/* post card */
6760	atom_asic_init(rdev->mode_info.atom_context);
6761
6762	/* init golden registers */
6763	si_init_golden_registers(rdev);
6764
6765	if (rdev->pm.pm_method == PM_METHOD_DPM)
6766		radeon_pm_resume(rdev);
6767
6768	rdev->accel_working = true;
6769	r = si_startup(rdev);
6770	if (r) {
6771		DRM_ERROR("si startup failed on resume\n");
6772		rdev->accel_working = false;
6773		return r;
6774	}
6775
6776	return r;
6777
6778}
6779
6780int si_suspend(struct radeon_device *rdev)
6781{
6782	radeon_pm_suspend(rdev);
6783	radeon_audio_fini(rdev);
6784	radeon_vm_manager_fini(rdev);
6785	si_cp_enable(rdev, false);
6786	cayman_dma_stop(rdev);
6787	if (rdev->has_uvd) {
6788		radeon_uvd_suspend(rdev);
6789		uvd_v1_0_fini(rdev);
 
6790	}
6791	if (rdev->has_vce)
6792		radeon_vce_suspend(rdev);
6793	si_fini_pg(rdev);
6794	si_fini_cg(rdev);
6795	si_irq_suspend(rdev);
6796	radeon_wb_disable(rdev);
6797	si_pcie_gart_disable(rdev);
6798	return 0;
6799}
6800
6801/* Plan is to move initialization in that function and use
6802 * helper function so that radeon_device_init pretty much
6803 * do nothing more than calling asic specific function. This
6804 * should also allow to remove a bunch of callback function
6805 * like vram_info.
6806 */
6807int si_init(struct radeon_device *rdev)
6808{
6809	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6810	int r;
6811
6812	/* Read BIOS */
6813	if (!radeon_get_bios(rdev)) {
6814		if (ASIC_IS_AVIVO(rdev))
6815			return -EINVAL;
6816	}
6817	/* Must be an ATOMBIOS */
6818	if (!rdev->is_atom_bios) {
6819		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6820		return -EINVAL;
6821	}
6822	r = radeon_atombios_init(rdev);
6823	if (r)
6824		return r;
6825
6826	/* Post card if necessary */
6827	if (!radeon_card_posted(rdev)) {
6828		if (!rdev->bios) {
6829			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6830			return -EINVAL;
6831		}
6832		DRM_INFO("GPU not posted. posting now...\n");
6833		atom_asic_init(rdev->mode_info.atom_context);
6834	}
6835	/* init golden registers */
6836	si_init_golden_registers(rdev);
6837	/* Initialize scratch registers */
6838	si_scratch_init(rdev);
6839	/* Initialize surface registers */
6840	radeon_surface_init(rdev);
6841	/* Initialize clocks */
6842	radeon_get_clock_info(rdev_to_drm(rdev));
6843
6844	/* Fence driver */
6845	radeon_fence_driver_init(rdev);
6846
6847	/* initialize memory controller */
6848	r = si_mc_init(rdev);
6849	if (r)
6850		return r;
6851	/* Memory manager */
6852	r = radeon_bo_init(rdev);
6853	if (r)
6854		return r;
6855
6856	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6857	    !rdev->rlc_fw || !rdev->mc_fw) {
6858		r = si_init_microcode(rdev);
6859		if (r) {
6860			DRM_ERROR("Failed to load firmware!\n");
6861			return r;
6862		}
6863	}
6864
6865	/* Initialize power management */
6866	radeon_pm_init(rdev);
6867
6868	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6869	ring->ring_obj = NULL;
6870	r600_ring_init(rdev, ring, 1024 * 1024);
6871
6872	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6873	ring->ring_obj = NULL;
6874	r600_ring_init(rdev, ring, 1024 * 1024);
6875
6876	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6877	ring->ring_obj = NULL;
6878	r600_ring_init(rdev, ring, 1024 * 1024);
6879
6880	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6881	ring->ring_obj = NULL;
6882	r600_ring_init(rdev, ring, 64 * 1024);
6883
6884	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6885	ring->ring_obj = NULL;
6886	r600_ring_init(rdev, ring, 64 * 1024);
6887
6888	si_uvd_init(rdev);
6889	si_vce_init(rdev);
6890
6891	rdev->ih.ring_obj = NULL;
6892	r600_ih_ring_init(rdev, 64 * 1024);
6893
6894	r = r600_pcie_gart_init(rdev);
6895	if (r)
6896		return r;
6897
6898	rdev->accel_working = true;
6899	r = si_startup(rdev);
6900	if (r) {
6901		dev_err(rdev->dev, "disabling GPU acceleration\n");
6902		si_cp_fini(rdev);
6903		cayman_dma_fini(rdev);
6904		si_irq_fini(rdev);
6905		sumo_rlc_fini(rdev);
6906		radeon_wb_fini(rdev);
6907		radeon_ib_pool_fini(rdev);
6908		radeon_vm_manager_fini(rdev);
6909		radeon_irq_kms_fini(rdev);
6910		si_pcie_gart_fini(rdev);
6911		rdev->accel_working = false;
6912	}
6913
6914	/* Don't start up if the MC ucode is missing.
6915	 * The default clocks and voltages before the MC ucode
6916	 * is loaded are not suffient for advanced operations.
6917	 */
6918	if (!rdev->mc_fw) {
6919		DRM_ERROR("radeon: MC ucode required for NI+.\n");
6920		return -EINVAL;
6921	}
6922
6923	return 0;
6924}
6925
6926void si_fini(struct radeon_device *rdev)
6927{
6928	radeon_pm_fini(rdev);
6929	si_cp_fini(rdev);
6930	cayman_dma_fini(rdev);
6931	si_fini_pg(rdev);
6932	si_fini_cg(rdev);
6933	si_irq_fini(rdev);
6934	sumo_rlc_fini(rdev);
6935	radeon_wb_fini(rdev);
6936	radeon_vm_manager_fini(rdev);
6937	radeon_ib_pool_fini(rdev);
6938	radeon_irq_kms_fini(rdev);
6939	if (rdev->has_uvd) {
6940		uvd_v1_0_fini(rdev);
6941		radeon_uvd_fini(rdev);
6942	}
6943	if (rdev->has_vce)
6944		radeon_vce_fini(rdev);
6945	si_pcie_gart_fini(rdev);
6946	r600_vram_scratch_fini(rdev);
6947	radeon_gem_fini(rdev);
6948	radeon_fence_driver_fini(rdev);
6949	radeon_bo_fini(rdev);
6950	radeon_atombios_fini(rdev);
6951	kfree(rdev->bios);
6952	rdev->bios = NULL;
6953}
6954
6955/**
6956 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6957 *
6958 * @rdev: radeon_device pointer
6959 *
6960 * Fetches a GPU clock counter snapshot (SI).
6961 * Returns the 64 bit clock counter snapshot.
6962 */
6963uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6964{
6965	uint64_t clock;
6966
6967	mutex_lock(&rdev->gpu_clock_mutex);
6968	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6969	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6970		((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6971	mutex_unlock(&rdev->gpu_clock_mutex);
6972	return clock;
6973}
6974
6975int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6976{
6977	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6978	int r;
6979
6980	/* bypass vclk and dclk with bclk */
6981	WREG32_P(CG_UPLL_FUNC_CNTL_2,
6982		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6983		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6984
6985	/* put PLL in bypass mode */
6986	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6987
6988	if (!vclk || !dclk) {
6989		/* keep the Bypass mode */
6990		return 0;
6991	}
6992
6993	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6994					  16384, 0x03FFFFFF, 0, 128, 5,
6995					  &fb_div, &vclk_div, &dclk_div);
6996	if (r)
6997		return r;
6998
6999	/* set RESET_ANTI_MUX to 0 */
7000	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7001
7002	/* set VCO_MODE to 1 */
7003	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7004
7005	/* disable sleep mode */
7006	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7007
7008	/* deassert UPLL_RESET */
7009	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7010
7011	mdelay(1);
7012
7013	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7014	if (r)
7015		return r;
7016
7017	/* assert UPLL_RESET again */
7018	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7019
7020	/* disable spread spectrum. */
7021	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7022
7023	/* set feedback divider */
7024	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7025
7026	/* set ref divider to 0 */
7027	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7028
7029	if (fb_div < 307200)
7030		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7031	else
7032		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7033
7034	/* set PDIV_A and PDIV_B */
7035	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7036		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7037		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7038
7039	/* give the PLL some time to settle */
7040	mdelay(15);
7041
7042	/* deassert PLL_RESET */
7043	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7044
7045	mdelay(15);
7046
7047	/* switch from bypass mode to normal mode */
7048	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7049
7050	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7051	if (r)
7052		return r;
7053
7054	/* switch VCLK and DCLK selection */
7055	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7056		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7057		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7058
7059	mdelay(100);
7060
7061	return 0;
7062}
7063
7064static void si_pcie_gen3_enable(struct radeon_device *rdev)
7065{
7066	struct pci_dev *root = rdev->pdev->bus->self;
7067	enum pci_bus_speed speed_cap;
7068	u32 speed_cntl, current_data_rate;
7069	int i;
7070	u16 tmp16;
7071
7072	if (pci_is_root_bus(rdev->pdev->bus))
7073		return;
7074
7075	if (radeon_pcie_gen2 == 0)
7076		return;
7077
7078	if (rdev->flags & RADEON_IS_IGP)
7079		return;
7080
7081	if (!(rdev->flags & RADEON_IS_PCIE))
7082		return;
7083
7084	speed_cap = pcie_get_speed_cap(root);
7085	if (speed_cap == PCI_SPEED_UNKNOWN)
7086		return;
7087
7088	if ((speed_cap != PCIE_SPEED_8_0GT) &&
7089	    (speed_cap != PCIE_SPEED_5_0GT))
7090		return;
7091
7092	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7093	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7094		LC_CURRENT_DATA_RATE_SHIFT;
7095	if (speed_cap == PCIE_SPEED_8_0GT) {
7096		if (current_data_rate == 2) {
7097			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7098			return;
7099		}
7100		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7101	} else if (speed_cap == PCIE_SPEED_5_0GT) {
7102		if (current_data_rate == 1) {
7103			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7104			return;
7105		}
7106		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7107	}
7108
7109	if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
7110		return;
7111
7112	if (speed_cap == PCIE_SPEED_8_0GT) {
7113		/* re-try equalization if gen3 is not already enabled */
7114		if (current_data_rate != 2) {
7115			u16 bridge_cfg, gpu_cfg;
7116			u16 bridge_cfg2, gpu_cfg2;
7117			u32 max_lw, current_lw, tmp;
7118
7119			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7120			pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
 
 
 
7121
7122			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7123			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7124			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7125
7126			if (current_lw < max_lw) {
7127				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7128				if (tmp & LC_RENEGOTIATION_SUPPORT) {
7129					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7130					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7131					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7132					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7133				}
7134			}
7135
7136			for (i = 0; i < 10; i++) {
7137				/* check status */
7138				pcie_capability_read_word(rdev->pdev,
7139							  PCI_EXP_DEVSTA,
7140							  &tmp16);
7141				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7142					break;
7143
7144				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7145							  &bridge_cfg);
7146				pcie_capability_read_word(rdev->pdev,
7147							  PCI_EXP_LNKCTL,
7148							  &gpu_cfg);
7149
7150				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7151							  &bridge_cfg2);
7152				pcie_capability_read_word(rdev->pdev,
7153							  PCI_EXP_LNKCTL2,
7154							  &gpu_cfg2);
7155
7156				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7157				tmp |= LC_SET_QUIESCE;
7158				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7159
7160				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7161				tmp |= LC_REDO_EQ;
7162				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7163
7164				msleep(100);
7165
7166				/* linkctl */
7167				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
7168								   PCI_EXP_LNKCTL_HAWD,
7169								   bridge_cfg &
7170								   PCI_EXP_LNKCTL_HAWD);
7171				pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,
7172								   PCI_EXP_LNKCTL_HAWD,
7173								   gpu_cfg &
7174								   PCI_EXP_LNKCTL_HAWD);
 
 
 
 
 
 
 
 
7175
7176				/* linkctl2 */
7177				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
7178								   PCI_EXP_LNKCTL2_ENTER_COMP |
7179								   PCI_EXP_LNKCTL2_TX_MARGIN,
7180								   bridge_cfg2 &
7181								   (PCI_EXP_LNKCTL2_ENTER_COMP |
7182								    PCI_EXP_LNKCTL2_TX_MARGIN));
7183				pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7184								   PCI_EXP_LNKCTL2_ENTER_COMP |
7185								   PCI_EXP_LNKCTL2_TX_MARGIN,
7186								   gpu_cfg2 &
7187								   (PCI_EXP_LNKCTL2_ENTER_COMP |
7188								    PCI_EXP_LNKCTL2_TX_MARGIN));
 
 
 
 
 
 
 
 
 
 
7189
7190				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7191				tmp &= ~LC_SET_QUIESCE;
7192				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7193			}
7194		}
7195	}
7196
7197	/* set the link speed */
7198	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7199	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7200	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7201
7202	tmp16 = 0;
 
7203	if (speed_cap == PCIE_SPEED_8_0GT)
7204		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
7205	else if (speed_cap == PCIE_SPEED_5_0GT)
7206		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
7207	else
7208		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
7209	pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7210					   PCI_EXP_LNKCTL2_TLS, tmp16);
7211
7212	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7213	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7214	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7215
7216	for (i = 0; i < rdev->usec_timeout; i++) {
7217		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7218		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7219			break;
7220		udelay(1);
7221	}
7222}
7223
7224static void si_program_aspm(struct radeon_device *rdev)
7225{
7226	u32 data, orig;
7227	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7228	bool disable_clkreq = false;
7229
7230	if (radeon_aspm == 0)
7231		return;
7232
7233	if (!(rdev->flags & RADEON_IS_PCIE))
7234		return;
7235
7236	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7237	data &= ~LC_XMIT_N_FTS_MASK;
7238	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7239	if (orig != data)
7240		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7241
7242	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7243	data |= LC_GO_TO_RECOVERY;
7244	if (orig != data)
7245		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7246
7247	orig = data = RREG32_PCIE(PCIE_P_CNTL);
7248	data |= P_IGNORE_EDB_ERR;
7249	if (orig != data)
7250		WREG32_PCIE(PCIE_P_CNTL, data);
7251
7252	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7253	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7254	data |= LC_PMI_TO_L1_DIS;
7255	if (!disable_l0s)
7256		data |= LC_L0S_INACTIVITY(7);
7257
7258	if (!disable_l1) {
7259		data |= LC_L1_INACTIVITY(7);
7260		data &= ~LC_PMI_TO_L1_DIS;
7261		if (orig != data)
7262			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7263
7264		if (!disable_plloff_in_l1) {
7265			bool clk_req_support;
7266
7267			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7268			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7269			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7270			if (orig != data)
7271				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7272
7273			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7274			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7275			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7276			if (orig != data)
7277				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7278
7279			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7280			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7281			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7282			if (orig != data)
7283				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7284
7285			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7286			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7287			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7288			if (orig != data)
7289				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7290
7291			if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7292				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7293				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7294				if (orig != data)
7295					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7296
7297				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7298				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7299				if (orig != data)
7300					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7301
7302				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7303				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7304				if (orig != data)
7305					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7306
7307				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7308				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7309				if (orig != data)
7310					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7311
7312				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7313				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7314				if (orig != data)
7315					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7316
7317				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7318				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7319				if (orig != data)
7320					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7321
7322				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7323				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7324				if (orig != data)
7325					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7326
7327				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7328				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7329				if (orig != data)
7330					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7331			}
7332			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7333			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7334			data |= LC_DYN_LANES_PWR_STATE(3);
7335			if (orig != data)
7336				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7337
7338			orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7339			data &= ~LS2_EXIT_TIME_MASK;
7340			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7341				data |= LS2_EXIT_TIME(5);
7342			if (orig != data)
7343				WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7344
7345			orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7346			data &= ~LS2_EXIT_TIME_MASK;
7347			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7348				data |= LS2_EXIT_TIME(5);
7349			if (orig != data)
7350				WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7351
7352			if (!disable_clkreq &&
7353			    !pci_is_root_bus(rdev->pdev->bus)) {
7354				struct pci_dev *root = rdev->pdev->bus->self;
7355				u32 lnkcap;
7356
7357				clk_req_support = false;
7358				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7359				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7360					clk_req_support = true;
7361			} else {
7362				clk_req_support = false;
7363			}
7364
7365			if (clk_req_support) {
7366				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7367				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7368				if (orig != data)
7369					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7370
7371				orig = data = RREG32(THM_CLK_CNTL);
7372				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7373				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7374				if (orig != data)
7375					WREG32(THM_CLK_CNTL, data);
7376
7377				orig = data = RREG32(MISC_CLK_CNTL);
7378				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7379				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7380				if (orig != data)
7381					WREG32(MISC_CLK_CNTL, data);
7382
7383				orig = data = RREG32(CG_CLKPIN_CNTL);
7384				data &= ~BCLK_AS_XCLK;
7385				if (orig != data)
7386					WREG32(CG_CLKPIN_CNTL, data);
7387
7388				orig = data = RREG32(CG_CLKPIN_CNTL_2);
7389				data &= ~FORCE_BIF_REFCLK_EN;
7390				if (orig != data)
7391					WREG32(CG_CLKPIN_CNTL_2, data);
7392
7393				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7394				data &= ~MPLL_CLKOUT_SEL_MASK;
7395				data |= MPLL_CLKOUT_SEL(4);
7396				if (orig != data)
7397					WREG32(MPLL_BYPASSCLK_SEL, data);
7398
7399				orig = data = RREG32(SPLL_CNTL_MODE);
7400				data &= ~SPLL_REFCLK_SEL_MASK;
7401				if (orig != data)
7402					WREG32(SPLL_CNTL_MODE, data);
7403			}
7404		}
7405	} else {
7406		if (orig != data)
7407			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7408	}
7409
7410	orig = data = RREG32_PCIE(PCIE_CNTL2);
7411	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7412	if (orig != data)
7413		WREG32_PCIE(PCIE_CNTL2, data);
7414
7415	if (!disable_l0s) {
7416		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7417		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7418			data = RREG32_PCIE(PCIE_LC_STATUS1);
7419			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7420				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7421				data &= ~LC_L0S_INACTIVITY_MASK;
7422				if (orig != data)
7423					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7424			}
7425		}
7426	}
7427}
7428
7429static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7430{
7431	unsigned i;
7432
7433	/* make sure VCEPLL_CTLREQ is deasserted */
7434	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7435
7436	mdelay(10);
7437
7438	/* assert UPLL_CTLREQ */
7439	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7440
7441	/* wait for CTLACK and CTLACK2 to get asserted */
7442	for (i = 0; i < 100; ++i) {
7443		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7444		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7445			break;
7446		mdelay(10);
7447	}
7448
7449	/* deassert UPLL_CTLREQ */
7450	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7451
7452	if (i == 100) {
7453		DRM_ERROR("Timeout setting UVD clocks!\n");
7454		return -ETIMEDOUT;
7455	}
7456
7457	return 0;
7458}
7459
7460int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7461{
7462	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7463	int r;
7464
7465	/* bypass evclk and ecclk with bclk */
7466	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7467		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7468		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7469
7470	/* put PLL in bypass mode */
7471	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7472		     ~VCEPLL_BYPASS_EN_MASK);
7473
7474	if (!evclk || !ecclk) {
7475		/* keep the Bypass mode, put PLL to sleep */
7476		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7477			     ~VCEPLL_SLEEP_MASK);
7478		return 0;
7479	}
7480
7481	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7482					  16384, 0x03FFFFFF, 0, 128, 5,
7483					  &fb_div, &evclk_div, &ecclk_div);
7484	if (r)
7485		return r;
7486
7487	/* set RESET_ANTI_MUX to 0 */
7488	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7489
7490	/* set VCO_MODE to 1 */
7491	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7492		     ~VCEPLL_VCO_MODE_MASK);
7493
7494	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
7495	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7496		     ~VCEPLL_SLEEP_MASK);
7497	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7498
7499	/* deassert VCEPLL_RESET */
7500	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7501
7502	mdelay(1);
7503
7504	r = si_vce_send_vcepll_ctlreq(rdev);
7505	if (r)
7506		return r;
7507
7508	/* assert VCEPLL_RESET again */
7509	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7510
7511	/* disable spread spectrum. */
7512	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7513
7514	/* set feedback divider */
7515	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7516
7517	/* set ref divider to 0 */
7518	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7519
7520	/* set PDIV_A and PDIV_B */
7521	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7522		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7523		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7524
7525	/* give the PLL some time to settle */
7526	mdelay(15);
7527
7528	/* deassert PLL_RESET */
7529	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7530
7531	mdelay(15);
7532
7533	/* switch from bypass mode to normal mode */
7534	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7535
7536	r = si_vce_send_vcepll_ctlreq(rdev);
7537	if (r)
7538		return r;
7539
7540	/* switch VCLK and DCLK selection */
7541	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7542		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7543		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7544
7545	mdelay(100);
7546
7547	return 0;
7548}